{
"metadata": {
"name": "VariantBench",
"evalCount": 118,
"modelCount": 31,
"trialCount": 10974,
"scoredTrialCount": 3950,
"source": "variantbench",
"generatedAt": "2026-07-16T20:55:16.115Z",
"facets": {
"broadCategory": [
{
"value": "Variant Discovery & QC",
"count": 48
},
{
"value": "Statistical & Population Genomics",
"count": 40
},
{
"value": "Personal & Clinical Genomics",
"count": 30
}
],
"subcategory": [
{
"value": "Small-Variant Discovery & QC",
"count": 16
},
{
"value": "Genotype & Sample QC",
"count": 14
},
{
"value": "SV, CNV & Repeat Discovery & QC",
"count": 12
},
{
"value": "Association & Molecular QTLs",
"count": 11
},
{
"value": "Fine-Mapping & Gene Prioritization",
"count": 11
},
{
"value": "Family, Inheritance & De Novo Variation",
"count": 9
},
{
"value": "Genetic Architecture & Prediction",
"count": 8
},
{
"value": "Causal Inference",
"count": 7
},
{
"value": "Read & Reference Processing",
"count": 6
},
{
"value": "Personal Functional Genomics",
"count": 6
},
{
"value": "Diagnosis & Pathogenicity",
"count": 6
},
{
"value": "Cancer Genomics & Prognosis",
"count": 5
},
{
"value": "Pharmacogenomics & Therapeutics",
"count": 4
},
{
"value": "Population Variation & Cohort Comparison",
"count": 3
}
],
"assayGroup": [
{
"value": "Other",
"count": 36
},
{
"value": "Short-read WGS",
"count": 35
},
{
"value": "Long-read WGS",
"count": 29
},
{
"value": "Exome sequencing",
"count": 18
}
],
"platform": [
{
"value": "Illumina",
"count": 58
},
{
"value": "Other",
"count": 33
},
{
"value": "PacBio",
"count": 23
},
{
"value": "ONT",
"count": 4
}
],
"diseaseCategory": [
{
"value": "no specific disease",
"count": 83
},
{
"value": "chronic disease",
"count": 17
},
{
"value": "cancer",
"count": 9
},
{
"value": "rare disease",
"count": 9
}
]
},
"exampleCount": 8,
"trajectoryCount": 28
},
"tags": {
"A4_rsid_merge_remap": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Read & Reference Processing",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "chronic disease"
},
"CNV_04": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Cancer Genomics & Prognosis",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "cancer"
},
"CNV_05": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Cancer Genomics & Prognosis",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "cancer"
},
"CNV_07": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "cancer"
},
"CNV_08": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Cancer Genomics & Prognosis",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "cancer"
},
"CNV_12": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Cancer Genomics & Prognosis",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "cancer"
},
"DR_01_t2d_polygenic_score": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Personal Functional Genomics",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "chronic disease"
},
"E3_detection_denominator_donor_availability": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Population Variation & Cohort Comparison",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"E8_cryptic_relatedness_effective_n": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"GWAS_02_strongest_locus_effect_vs_significance": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"GWAS_08": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"GWAS_12": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"PGX01_conservative_validation_pgx_summary": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Pharmacogenomics & Therapeutics",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"Q1b_vntr_contraction_chr19": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"Q2_D2_amy1_mappability": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Read & Reference Processing",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"Q3_D3_chm13_divergence_snps_chr19": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"QC_05_dual_indel_consensus": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"RFC1_01": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Diagnosis & Pathogenicity",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "rare disease"
},
"WES_C_gc_confounded_heterozygosity": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"WG_01_reference_absent_overclaim": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Read & Reference Processing",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"alignment_and_preprocessing_pedigree_relatedness_verification": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"annotation_and_interpretation_pathway_drug_target_triage": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Pharmacogenomics & Therapeutics",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "chronic disease"
},
"annotation_and_interpretation_pgx_star_allele_phasing": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Pharmacogenomics & Therapeutics",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"cis_mr_bcaa": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Causal Inference",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"cis_mr_ld_confound": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Causal Inference",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"cis_mr_opposite": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Causal Inference",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"coloc_hmgcr": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"coloc_hmgcr_splice": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"coloc_ldl_liver_data_quality": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"eqtl_cis_window_build_mismatch": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Read & Reference Processing",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"eqtl_context_effect_size_attenuation_not_portability": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"eqtl_locus_robustness_cross_mappability_filter": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"eqtl_trans_target_count_multiple_testing": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"fine_mapping_effector_gene_prioritization": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "chronic disease"
},
"gene_family_dosage_single": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Personal Functional Genomics",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"genetic_correlation_meta_eur_traits": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"glucose_finemap_ld_mismatch": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_02_independent_signals_vs_distinct_loci": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_24_gene_mapping_prioritization_signal_density": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_25_causal_variant_fine_mapping_resolution": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_28_prs_prediction_accuracy_validation": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_30_cross_ancestry_heritability_reference_mismatch": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_33_acan_supported_variant_sets": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_36_skeletal_growth_regulatory_genes": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"gwas_association_module_vs_individual_robustness": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"gwas_chry_nonpar_female_phantom_carriers": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"h20_context_sample_size_not_biology": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"h20_trans_targets_not_effector_gene": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "chronic disease"
},
"heritability_hdl_size_ldsc_estimator_meta_eur": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"heritability_hdl_size_meta_eur": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"heritability_hdl_size_pvalue_meta_eur": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Genetic Architecture & Prediction",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"heterogeneity_model_choice_bcaa_chr4": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"humanvar_07_population_scale_function_blindspot": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Pharmacogenomics & Therapeutics",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"ld_reference_eur_dedup": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Read & Reference Processing",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "no specific disease"
},
"male_chrx_nonpar_false_het": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"mendelian_randomization_trans_qtl_bystander_vs_causal": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Causal Inference",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "chronic disease"
},
"missing_homref": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Population Variation & Cohort Comparison",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"mouse_annotation_association_lineage_confounded_svs": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"mouse_annotation_vep_lof_count": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"mouse_filtering_inbred_singleton_indel_count": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"mouse_variant_calling_inbred_heterozygous_call_reliability": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"mr_glucose_t2d_genomewide": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Causal Inference",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"mt_diploid_allele_count": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"multiomics_mendelian_incidental_carrier_vs_affected": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Diagnosis & Pathogenicity",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "rare disease"
},
"multiomics_phased_compound_het_biallelic_knockout": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"multiomics_rna_editing_genuine_vs_snp_artifact": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Personal Functional Genomics",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"n070_meta_betas_log_vs_linear": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Association & Molecular QTLs",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"ont_rfc1_repeat_expansion_pathogenicity": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Diagnosis & Pathogenicity",
"assayGroup": "Long-read WGS",
"platform": "ONT",
"diseaseCategory": "rare disease"
},
"ont_xylt1_repeat_expansion_transmission_risk": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Long-read WGS",
"platform": "ONT",
"diseaseCategory": "rare disease"
},
"plasma_lactate_causal": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Causal Inference",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"plasma_lactate_hypothesis_gen_blind": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Causal Inference",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"plasma_lactate_variant_id": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Fine-Mapping & Gene Prioritization",
"assayGroup": "Other",
"platform": "Other",
"diseaseCategory": "chronic disease"
},
"pmid26547235_batch_effect_dna_repair_14": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Diagnosis & Pathogenicity",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "rare disease"
},
"pmid26547235_callable_region_denominator_04": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26547235_chry_father_son_concordance_11": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26547235_chry_hemizygous_male_10": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26547235_family_denovo_snv_01": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26547235_female_chry_absence_12_hard": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26547235_microdeletion_survey_het_02": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "rare disease"
},
"pmid26547235_mnp_blind_spot_09": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26547235_mother_chrx_son_transmission_13": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26547235_recessive_vs_denovo_05": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "rare disease"
},
"pmid26771485_motif_norm_05": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Personal Functional Genomics",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid26771485_size_window_03": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Personal Functional Genomics",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"pmid40670386_ont_systematic_errors_07": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"pmid40670386_strand_bias_01": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"q9_t1_phasing_acad11": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Personal Functional Genomics",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"qc_04_chr16_segdup_common_snp_count": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Diagnosis & Pathogenicity",
"assayGroup": "Other",
"platform": "Illumina",
"diseaseCategory": "rare disease"
},
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Diagnosis & Pathogenicity",
"assayGroup": "Other",
"platform": "Illumina",
"diseaseCategory": "rare disease"
},
"smaht_long_read_indel_filter_06": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"small_variant_caller_analysis": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"small_variant_calling_mnv_codon_consequence": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"small_variant_error_interp": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "ONT",
"diseaseCategory": "no specific disease"
},
"small_variant_inheritance": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "ONT",
"diseaseCategory": "no specific disease"
},
"somatic_sv_derivation_force_genotype_vs_subtract": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "cancer"
},
"specialized_germline_analyses_hla_mhc_call_reliability": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Read & Reference Processing",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"sv_chromosome_end_signal": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"sv_population_fst_top_differentiated_chr20": {
"broadCategory": "Statistical & Population Genomics",
"subcategory": "Population Variation & Cohort Comparison",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"sv_size_peak_alu": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"sv_size_peak_line": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"tr_inheritance_consistency": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"tr_structure_proportions": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"variant_01_trio_denovo_parental_confirmation": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"variant_02_trio_mendelian_error_rate_missing_handling": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"variant_03_trio_sv_denovo_parental_confirmation": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Family, Inheritance & De Novo Variation",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"variant_04_twin_genome_concordance_artifact": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"variant_06_homozygous_nonparental_allele_artifact": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"variant_07_structural_variant_size_threshold_overcount": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Short-read WGS",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"variant_cohort_region_depth_duplication_carriers": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "PacBio",
"diseaseCategory": "no specific disease"
},
"variant_filtering_and_benchmarking_child_homozygous_nonparental": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Genotype & Sample QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"variant_filtering_and_benchmarking_lof_artifact_triage": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "Small-Variant Discovery & QC",
"assayGroup": "Exome sequencing",
"platform": "Illumina",
"diseaseCategory": "no specific disease"
},
"variantbench_mmej_deletion_fraction": {
"broadCategory": "Personal & Clinical Genomics",
"subcategory": "Cancer Genomics & Prognosis",
"assayGroup": "Other",
"platform": "Illumina",
"diseaseCategory": "cancer"
},
"variantbench_multiplatform_focal_amplification": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "Illumina",
"diseaseCategory": "cancer"
},
"variantbench_sv_platform_reconciliation": {
"broadCategory": "Variant Discovery & QC",
"subcategory": "SV, CNV & Repeat Discovery & QC",
"assayGroup": "Long-read WGS",
"platform": "Illumina",
"diseaseCategory": "cancer"
}
},
"modelResults": [
{
"id": "opus-5-claude-code",
"modelName": "opus-5",
"modelLabel": "Opus 5",
"provider": "Anthropic",
"harness": "claude-code",
"harnessLabel": "Claude Code",
"config": "Opus 5 / Claude Code",
"passRate": 49.72,
"ci": [
41.39,
58.04
],
"nEvals": 118,
"correct": 176,
"gradeable": 354,
"runs": 354,
"avgCost": 2.78,
"totalCost": 936.78,
"evalBreakdown": {
"A4_rsid_merge_remap": [
3,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
1,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
2,
3
],
"CNV_12": [
3,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
1,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
3,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
1,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
2,
3
],
"cis_mr_opposite": [
1,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
2,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
1,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
3,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
2,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
1,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
2,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
3,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
3,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
2,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"q9_t1_phasing_acad11": [
3,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
1,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
2,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
1,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
3,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
3,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
3,
3
],
"sv_size_peak_line": [
3,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
2,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
1,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
2,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
1,
3
]
}
},
{
"id": "opus-5-pi",
"modelName": "opus-5",
"modelLabel": "Opus 5",
"provider": "Anthropic",
"harness": "pi",
"harnessLabel": "Pi",
"config": "Opus 5 / Pi",
"passRate": 49.44,
"ci": [
40.89,
57.98
],
"nEvals": 118,
"correct": 175,
"gradeable": 354,
"runs": 354,
"avgCost": 1.08,
"totalCost": 381.68,
"evalBreakdown": {
"A4_rsid_merge_remap": [
3,
3
],
"CNV_04": [
1,
3
],
"CNV_05": [
1,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
3,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
3,
3
],
"cis_mr_opposite": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
1,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
2,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
2,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
2,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
3,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
3,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
2,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
3,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
2,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
2,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"plasma_lactate_causal": [
3,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"q9_t1_phasing_acad11": [
1,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
1,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
2,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
3,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
3,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
3,
3
],
"sv_size_peak_line": [
3,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
1,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
1,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
]
}
},
{
"id": "grok-4-6-grok-build",
"modelName": "grok-4.6",
"modelLabel": "Grok-4.6",
"provider": "xAI",
"harness": "grok_build",
"harnessLabel": "Grok Build",
"config": "Grok-4.6 / Grok Build",
"passRate": 46.0,
"ci": [
37.8,
54.3
],
"nEvals": 118,
"correct": 163,
"gradeable": 354,
"runs": 354,
"avgCost": 0.98,
"totalCost": 343.73,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
3,
3
],
"cis_mr_ld_confound": [
1,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
3,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
0,
3
],
"E8_cryptic_relatedness_effective_n": [
1,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
1,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
2,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
3,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
1,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
1,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
2,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
2,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
0,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
2,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
3,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
1,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
1,
3
],
"pmid26771485_motif_norm_05": [
2,
3
],
"pmid26771485_size_window_03": [
0,
3
],
"pmid40670386_ont_systematic_errors_07": [
1,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
2,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
3,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
2,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
1,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
3,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
3,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
1,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
1,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "gpt-5-6-sol-openai-codex",
"modelName": "gpt-5.6-sol",
"modelLabel": "GPT-5.6 Sol",
"provider": "OpenAI",
"harness": "openai-codex",
"harnessLabel": "Codex",
"config": "GPT-5.6 Sol / Codex",
"passRate": 42.1,
"ci": [
33.8,
50.4
],
"nEvals": 118,
"correct": 149,
"gradeable": 354,
"runs": 354,
"avgCost": 2.38,
"totalCost": 841.47,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
3,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
2,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
0,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
2,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
2,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
0,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
2,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
2,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
2,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
1,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
2,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
1,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
1,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
3,
3
],
"q9_t1_phasing_acad11": [
1,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
3,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
3,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
2,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
2,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
2,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
3,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "claude-opus-4-8-pi",
"modelName": "claude-opus-4-8",
"modelLabel": "Claude Opus 4.8",
"provider": "Anthropic",
"harness": "pi",
"harnessLabel": "PI",
"config": "Claude Opus 4.8 / PI",
"passRate": 42.1,
"ci": [
33.8,
50.4
],
"nEvals": 118,
"correct": 149,
"gradeable": 354,
"runs": 354,
"avgCost": 3.12,
"totalCost": 1103.39,
"evalBreakdown": {
"A4_rsid_merge_remap": [
1,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
1,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
2,
3
],
"CNV_08": [
2,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
1,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
2,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
2,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
3,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
2,
3
],
"heritability_hdl_size_meta_eur": [
1,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
0,
3
],
"humanvar_07_population_scale_function_blindspot": [
0,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
3,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
1,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
1,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
1,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
2,
3
],
"pmid40670386_strand_bias_01": [
1,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
3,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
2,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
1,
3
],
"sv_population_fst_top_differentiated_chr20": [
1,
3
],
"sv_size_peak_alu": [
3,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
2,
3
],
"variantbench_multiplatform_focal_amplification": [
1,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "gpt-5-6-sol-pi",
"modelName": "gpt-5.6-sol",
"modelLabel": "GPT-5.6 Sol",
"provider": "OpenAI",
"harness": "pi",
"harnessLabel": "PI",
"config": "GPT-5.6 Sol / PI",
"passRate": 41.2,
"ci": [
32.8,
49.6
],
"nEvals": 118,
"correct": 146,
"gradeable": 354,
"runs": 354,
"avgCost": 2.71,
"totalCost": 953.27,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
2,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
0,
3
],
"CNV_04": [
0,
3
],
"CNV_05": [
1,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
2,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
1,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
2,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
2,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
1,
3
],
"heritability_hdl_size_meta_eur": [
1,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
1,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
1,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
3,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
0,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
3,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
2,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
3,
3
],
"q9_t1_phasing_acad11": [
1,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
1,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
3,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
3,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
3,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
2,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
3,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
1,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
0,
3
]
}
},
{
"id": "claude-opus-4-8-mini-swe-agent",
"modelName": "claude-opus-4-8",
"modelLabel": "Claude Opus 4.8",
"provider": "Anthropic",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "Claude Opus 4.8 / mini-SWE",
"passRate": 40.7,
"ci": [
33,
48.4
],
"nEvals": 118,
"correct": 144,
"gradeable": 354,
"runs": 354,
"avgCost": 2.72,
"totalCost": 954.76,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
1,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
1,
3
],
"CNV_05": [
1,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
2,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
2,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
3,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
2,
3
],
"heritability_hdl_size_meta_eur": [
1,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
1,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
2,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
1,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
1,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
2,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
1,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
1,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
2,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
2,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
1,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
2,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
1,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
1,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
2,
3
],
"variantbench_mmej_deletion_fraction": [
1,
3
],
"variantbench_multiplatform_focal_amplification": [
1,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "claude-opus-4-8-claude-code",
"modelName": "claude-opus-4-8",
"modelLabel": "Claude Opus 4.8",
"provider": "Anthropic",
"harness": "claude-code",
"harnessLabel": "Claude Code",
"config": "Claude Opus 4.8 / Claude Code",
"passRate": 39.8,
"ci": [
32,
47.6
],
"nEvals": 118,
"correct": 141,
"gradeable": 354,
"runs": 354,
"avgCost": 3.38,
"totalCost": 1167.8,
"evalBreakdown": {
"A4_rsid_merge_remap": [
2,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
2,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
1,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
1,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
1,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
1,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
1,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
1,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
2,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
2,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
1,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
2,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
3,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
2,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
2,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
2,
3
],
"q9_t1_phasing_acad11": [
1,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
1,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
3,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
2,
3
],
"sv_chromosome_end_signal": [
1,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
2,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
1,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
1,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "claude-sonnet-5-claude-code",
"modelName": "claude-sonnet-5",
"modelLabel": "Claude Sonnet 5",
"provider": "Anthropic",
"harness": "claude-code",
"harnessLabel": "Claude Code",
"config": "Claude Sonnet 5 / Claude Code",
"passRate": 38.7,
"ci": [
30.7,
46.7
],
"nEvals": 118,
"correct": 137,
"gradeable": 354,
"runs": 354,
"avgCost": 2.97,
"totalCost": 1041.46,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
1,
3
],
"CNV_08": [
2,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
1,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
1,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
3,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
3,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
0,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
0,
3
],
"heritability_hdl_size_meta_eur": [
2,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
0,
3
],
"humanvar_07_population_scale_function_blindspot": [
0,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
3,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
1,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
2,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
1,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
2,
3
],
"q9_t1_phasing_acad11": [
2,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
1,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
2,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
1,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
1,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "gpt-5-6-terra-openai-codex",
"modelName": "gpt-5.6-terra",
"modelLabel": "GPT-5.6 Terra",
"provider": "OpenAI",
"harness": "openai-codex",
"harnessLabel": "Codex",
"config": "GPT-5.6 Terra / Codex",
"passRate": 38.7,
"ci": [
30.4,
47
],
"nEvals": 118,
"correct": 137,
"gradeable": 354,
"runs": 354,
"avgCost": 1.27,
"totalCost": 449.96,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
3,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
1,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
3,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
2,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
0,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
3,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
2,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
3,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
0,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
1,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
2,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
0,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
3,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
1,
3
],
"pmid26771485_size_window_03": [
2,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
1,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
3,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
1,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
3,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
2,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
2,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
1,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
2,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "grok-4-6-pi",
"modelName": "grok-4.6",
"modelLabel": "Grok 4.6",
"provider": "xAI",
"harness": "pi",
"harnessLabel": "PI",
"config": "Grok 4.6 / PI",
"passRate": 38.1,
"ci": [
30.1,
46.1
],
"nEvals": 118,
"correct": 135,
"gradeable": 354,
"runs": 354,
"avgCost": 0.49,
"totalCost": 170.32,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
1,
3
],
"cis_mr_ld_confound": [
1,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
3,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
1,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
1,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
0,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
2,
3
],
"heritability_hdl_size_meta_eur": [
1,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
0,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
1,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
2,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
1,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
2,
3
],
"sv_size_peak_line": [
1,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
2,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
2,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
0,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
2,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
2,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "claude-sonnet-5-mini-swe-agent",
"modelName": "claude-sonnet-5",
"modelLabel": "Claude Sonnet 5",
"provider": "Anthropic",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "Claude Sonnet 5 / mini-SWE",
"passRate": 38.1,
"ci": [
30.3,
46
],
"nEvals": 118,
"correct": 135,
"gradeable": 354,
"runs": 354,
"avgCost": 2.18,
"totalCost": 770.47,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
0,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
1,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
1,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
2,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
2,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
3,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
2,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
0,
3
],
"h20_trans_targets_not_effector_gene": [
2,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
0,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
1,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
3,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
2,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
1,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
1,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
0,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
2,
3
],
"q9_t1_phasing_acad11": [
3,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
1,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
1,
3
],
"sv_population_fst_top_differentiated_chr20": [
1,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
1,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
1,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
2,
3
],
"variantbench_mmej_deletion_fraction": [
1,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
1,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "grok-4-5-mini-swe-agent",
"modelName": "grok-4.5",
"modelLabel": "Grok 4.5",
"provider": "xAI",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "Grok 4.5 / mini-SWE",
"passRate": 38.1,
"ci": [
30.1,
46.1
],
"nEvals": 118,
"correct": 135,
"gradeable": 354,
"runs": 354,
"avgCost": 0.49,
"totalCost": 174.8,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
2,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
0,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
2,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
2,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
1,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
1,
3
],
"h20_trans_targets_not_effector_gene": [
1,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
2,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
1,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
1,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
2,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
1,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
2,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
3,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
1,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
3,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
1,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
2,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
3,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
1,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
1,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
2,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "claude-sonnet-5-pi",
"modelName": "claude-sonnet-5",
"modelLabel": "Claude Sonnet 5",
"provider": "Anthropic",
"harness": "pi",
"harnessLabel": "PI",
"config": "Claude Sonnet 5 / PI",
"passRate": 38.1,
"ci": [
30.5,
45.7
],
"nEvals": 118,
"correct": 135,
"gradeable": 354,
"runs": 354,
"avgCost": 1.95,
"totalCost": 687.31,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
1,
3
],
"cis_mr_opposite": [
0,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
1,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
2,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
2,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
2,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
1,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
2,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
1,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
2,
3
],
"heritability_hdl_size_meta_eur": [
2,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
0,
3
],
"humanvar_07_population_scale_function_blindspot": [
2,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
3,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
3,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
2,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
1,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
1,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
2,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
1,
3
],
"Q1b_vntr_contraction_chr19": [
1,
3
],
"Q2_D2_amy1_mappability": [
1,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
2,
3
],
"q9_t1_phasing_acad11": [
2,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
1,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
1,
3
],
"variantbench_mmej_deletion_fraction": [
2,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
1,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "grok-4-5-pi",
"modelName": "grok-4.5",
"modelLabel": "Grok 4.5",
"provider": "xAI",
"harness": "pi",
"harnessLabel": "PI",
"config": "Grok 4.5 / PI",
"passRate": 37.6,
"ci": [
29.6,
45.5
],
"nEvals": 118,
"correct": 133,
"gradeable": 354,
"runs": 354,
"avgCost": 0.46,
"totalCost": 160.84,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
2,
3
],
"CNV_07": [
2,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
1,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
1,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
1,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
0,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
1,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
0,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
1,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
2,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
2,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
2,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
2,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
2,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
3,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
3,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
1,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
2,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
2,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
3,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
1,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
1,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
1,
3
],
"variantbench_mmej_deletion_fraction": [
1,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "claude-opus-4-7-pi",
"modelName": "claude-opus-4-7",
"modelLabel": "Claude Opus 4.7",
"provider": "Anthropic",
"harness": "pi",
"harnessLabel": "PI",
"config": "Claude Opus 4.7 / PI",
"passRate": 37,
"ci": [
28.7,
45.3
],
"nEvals": 118,
"correct": 131,
"gradeable": 354,
"runs": 354,
"avgCost": 2.33,
"totalCost": 809.88,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
1,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
0,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
1,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
0,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
1,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
3,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
2,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
2,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
1,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
2,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
1,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
1,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
2,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
2,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "claude-opus-4-7-claude-code",
"modelName": "claude-opus-4-7",
"modelLabel": "Claude Opus 4.7",
"provider": "Anthropic",
"harness": "claude-code",
"harnessLabel": "Claude Code",
"config": "Claude Opus 4.7 / Claude Code",
"passRate": 35.6,
"ci": [
27.7,
43.5
],
"nEvals": 118,
"correct": 126,
"gradeable": 354,
"runs": 354,
"avgCost": 2.73,
"totalCost": 943.18,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
1,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
2,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
1,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
1,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
2,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
1,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
2,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
2,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
2,
3
],
"n070_meta_betas_log_vs_linear": [
2,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
1,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
1,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
2,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
1,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "gemini-3-5-flash-pi",
"modelName": "gemini-3.5-flash",
"modelLabel": "Gemini 3.5 Flash",
"provider": "Google",
"harness": "pi",
"harnessLabel": "PI",
"config": "Gemini 3.5 Flash / PI",
"passRate": 35.3,
"ci": [
27.7,
42.9
],
"nEvals": 118,
"correct": 125,
"gradeable": 354,
"runs": 354,
"avgCost": 1.57,
"totalCost": 556.7,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
2,
3
],
"CNV_08": [
2,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
2,
3
],
"eqtl_cis_window_build_mismatch": [
2,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
1,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
2,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
1,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
0,
3
],
"heritability_hdl_size_meta_eur": [
2,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
1,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
2,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
1,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
3,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
2,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
2,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
2,
3
],
"pmid26771485_size_window_03": [
0,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
3,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
2,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
1,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
1,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
3,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
1,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
1,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
1,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
1,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "claude-opus-4-7-mini-swe-agent",
"modelName": "claude-opus-4-7",
"modelLabel": "Claude Opus 4.7",
"provider": "Anthropic",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "Claude Opus 4.7 / mini-SWE",
"passRate": 35,
"ci": [
26.9,
43.2
],
"nEvals": 118,
"correct": 124,
"gradeable": 354,
"runs": 354,
"avgCost": 2.78,
"totalCost": 978.65,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
0,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
3,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
1,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
1,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
2,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
1,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
2,
3
],
"h20_trans_targets_not_effector_gene": [
2,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
1,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
3,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
1,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
0,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
2,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
2,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
1,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
2,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
1,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
3,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "gpt-5-6-terra-pi",
"modelName": "gpt-5.6-terra",
"modelLabel": "GPT-5.6 Terra",
"provider": "OpenAI",
"harness": "pi",
"harnessLabel": "PI",
"config": "GPT-5.6 Terra / PI",
"passRate": 34.7,
"ci": [
27,
42.5
],
"nEvals": 118,
"correct": 123,
"gradeable": 354,
"runs": 354,
"avgCost": 1.82,
"totalCost": 635.89,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
1,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
2,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
1,
3
],
"CNV_12": [
2,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
1,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
2,
3
],
"E8_cryptic_relatedness_effective_n": [
3,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
1,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
2,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
1,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
1,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
0,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
0,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
2,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
3,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
0,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
1,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
3,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
2,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
1,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
0,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
1,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
3,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
3,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
2,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
1,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
0,
3
]
}
},
{
"id": "gemini-3-1-pro-preview-pi",
"modelName": "gemini-3.1-pro-preview",
"modelLabel": "Gemini 3.1 Pro",
"provider": "Google",
"harness": "pi",
"harnessLabel": "PI",
"config": "Gemini 3.1 Pro / PI",
"passRate": 34.5,
"ci": [
27,
42
],
"nEvals": 118,
"correct": 122,
"gradeable": 354,
"runs": 354,
"avgCost": 1.01,
"totalCost": 359.19,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
0,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
1,
3
],
"CNV_04": [
0,
3
],
"CNV_05": [
2,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
2,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
2,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
1,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
1,
3
],
"h20_trans_targets_not_effector_gene": [
2,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
1,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
2,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
2,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
2,
3
],
"pmid26771485_motif_norm_05": [
1,
3
],
"pmid26771485_size_window_03": [
1,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
2,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
2,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
1,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
1,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
2,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
2,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
1,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
2,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
1,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "gemini-3-1-pro-preview-mini-swe-agent",
"modelName": "gemini-3.1-pro-preview",
"modelLabel": "Gemini 3.1 Pro",
"provider": "Google",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "Gemini 3.1 Pro / mini-SWE",
"passRate": 33.1,
"ci": [
25.2,
40.9
],
"nEvals": 118,
"correct": 117,
"gradeable": 354,
"runs": 354,
"avgCost": 1.27,
"totalCost": 450.75,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
0,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
1,
3
],
"CNV_04": [
0,
3
],
"CNV_05": [
2,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
1,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
2,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
3,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
0,
3
],
"h20_trans_targets_not_effector_gene": [
3,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
2,
3
],
"humanvar_07_population_scale_function_blindspot": [
0,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
3,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
1,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
3,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
2,
3
],
"pmid26771485_motif_norm_05": [
2,
3
],
"pmid26771485_size_window_03": [
0,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
1,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
1,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
1,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
2,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
1,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
2,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
0,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "kimi-k3-pi",
"modelName": "kimi-k3",
"modelLabel": "Kimi K3",
"provider": "Moonshot",
"harness": "pi",
"harnessLabel": "PI",
"config": "Kimi K3 / PI",
"passRate": 32.8,
"ci": [
25.1,
40.4
],
"nEvals": 118,
"correct": 116,
"gradeable": 354,
"runs": 354,
"avgCost": 1.04,
"totalCost": 363.3,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
3,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
1,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
2,
3
],
"E8_cryptic_relatedness_effective_n": [
1,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
1,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
1,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
1,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
2,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
1,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
1,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
1,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
2,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
2,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
2,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
0,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
2,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
2,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
2,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
2,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
2,
3
],
"variantbench_mmej_deletion_fraction": [
1,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
1,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
},
{
"id": "gemini-3-5-flash-mini-swe-agent",
"modelName": "gemini-3.5-flash",
"modelLabel": "Gemini 3.5 Flash",
"provider": "Google",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "Gemini 3.5 Flash / mini-SWE",
"passRate": 32.5,
"ci": [
24.7,
40.2
],
"nEvals": 118,
"correct": 115,
"gradeable": 354,
"runs": 354,
"avgCost": 2.89,
"totalCost": 994.64,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
2,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
0,
3
],
"CNV_04": [
2,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
1,
3
],
"CNV_08": [
2,
3
],
"CNV_12": [
2,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
2,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
2,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
1,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
3,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
1,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
1,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
0,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
1,
3
],
"h20_trans_targets_not_effector_gene": [
2,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
0,
3
],
"humanvar_07_population_scale_function_blindspot": [
0,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
3,
3
],
"missing_homref": [
3,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
0,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
3,
3
],
"n070_meta_betas_log_vs_linear": [
3,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
3,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
1,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
0,
3
],
"pmid26771485_size_window_03": [
0,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
1,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
2,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
1,
3
],
"smaht_long_read_indel_filter_06": [
3,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
2,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
3,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
1,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
1,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "gpt-5-5-mini-swe-agent",
"modelName": "gpt-5.5",
"modelLabel": "GPT-5.5",
"provider": "OpenAI",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "GPT-5.5 / mini-SWE",
"passRate": 30.2,
"ci": [
23.1,
37.3
],
"nEvals": 118,
"correct": 107,
"gradeable": 354,
"runs": 354,
"avgCost": 1.13,
"totalCost": 400.47,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
1,
3
],
"CNV_04": [
1,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
1,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
1,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
2,
3
],
"eqtl_trans_target_count_multiple_testing": [
1,
3
],
"fine_mapping_effector_gene_prioritization": [
2,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
1,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
1,
3
],
"heritability_hdl_size_meta_eur": [
1,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
2,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
1,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
0,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
2,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
2,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
2,
3
],
"n070_meta_betas_log_vs_linear": [
2,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
1,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
1,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
2,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
1,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
3,
3
],
"pmid26771485_motif_norm_05": [
1,
3
],
"pmid26771485_size_window_03": [
1,
3
],
"pmid40670386_ont_systematic_errors_07": [
1,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
1,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "gpt-5-6-luna-openai-codex",
"modelName": "gpt-5.6-luna",
"modelLabel": "GPT-5.6 Luna",
"provider": "OpenAI",
"harness": "openai-codex",
"harnessLabel": "Codex",
"config": "GPT-5.6 Luna / Codex",
"passRate": 29.9,
"ci": [
22.6,
37.3
],
"nEvals": 118,
"correct": 106,
"gradeable": 354,
"runs": 354,
"avgCost": 0.81,
"totalCost": 287.76,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
2,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
0,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
1,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
2,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
2,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
1,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
0,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
3,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
1,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
2,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
1,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
2,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
2,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
0,
3
],
"n070_meta_betas_log_vs_linear": [
0,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
3,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
1,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
1,
3
],
"pmid26771485_motif_norm_05": [
0,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
1,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
0,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
2,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
1,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
3,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
2,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
1,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
1,
3
]
}
},
{
"id": "gpt-5-5-pi",
"modelName": "gpt-5.5",
"modelLabel": "GPT-5.5",
"provider": "OpenAI",
"harness": "pi",
"harnessLabel": "PI",
"config": "GPT-5.5 / PI",
"passRate": 29.7,
"ci": [
22.3,
37
],
"nEvals": 118,
"correct": 105,
"gradeable": 354,
"runs": 354,
"avgCost": 1.32,
"totalCost": 466.36,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
1,
3
],
"CNV_04": [
0,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
1,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
3,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
1,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
2,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
2,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
1,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
1,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
3,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
0,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
1,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
2,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
2,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
2,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
1,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
3,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
1,
3
],
"pmid26771485_motif_norm_05": [
3,
3
],
"pmid26771485_size_window_03": [
0,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
1,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
1,
3
],
"q9_t1_phasing_acad11": [
2,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
1,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
2,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
1,
3
]
}
},
{
"id": "gpt-5-5-openai-codex",
"modelName": "gpt-5.5",
"modelLabel": "GPT-5.5",
"provider": "OpenAI",
"harness": "openai-codex",
"harnessLabel": "Codex",
"config": "GPT-5.5 / Codex",
"passRate": 28.8,
"ci": [
21.6,
36
],
"nEvals": 118,
"correct": 102,
"gradeable": 354,
"runs": 354,
"avgCost": 1.7,
"totalCost": 603.03,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
1,
3
],
"CNV_04": [
1,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
0,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
3,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
1,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
1,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
1,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
2,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
1,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
1,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
0,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
2,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
2,
3
],
"n070_meta_betas_log_vs_linear": [
1,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
2,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
2,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
2,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
2,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
1,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
1,
3
],
"pmid26771485_motif_norm_05": [
0,
3
],
"pmid26771485_size_window_03": [
0,
3
],
"pmid40670386_ont_systematic_errors_07": [
1,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
3,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
2,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
1,
3
],
"smaht_long_read_indel_filter_06": [
2,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
1,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
3,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
1,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
0,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "gpt-5-6-luna-pi",
"modelName": "gpt-5.6-luna",
"modelLabel": "GPT-5.6 Luna",
"provider": "OpenAI",
"harness": "pi",
"harnessLabel": "PI",
"config": "GPT-5.6 Luna / PI",
"passRate": 28.2,
"ci": [
20.8,
35.7
],
"nEvals": 118,
"correct": 100,
"gradeable": 354,
"runs": 354,
"avgCost": 0.78,
"totalCost": 267.91,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
3,
3
],
"cis_mr_bcaa": [
1,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
2,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
0,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
0,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
2,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
3,
3
],
"E3_detection_denominator_donor_availability": [
3,
3
],
"E8_cryptic_relatedness_effective_n": [
2,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
0,
3
],
"gene_family_dosage_single": [
3,
3
],
"genetic_correlation_meta_eur_traits": [
3,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
1,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
0,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
3,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
3,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
0,
3
],
"humanvar_07_population_scale_function_blindspot": [
3,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
1,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
3,
3
],
"mouse_annotation_vep_lof_count": [
2,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
0,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
0,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
3,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
0,
3
],
"n070_meta_betas_log_vs_linear": [
0,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
1,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
3,
3
],
"plasma_lactate_variant_id": [
3,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
1,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
1,
3
],
"pmid26771485_motif_norm_05": [
1,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
0,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
2,
3
],
"q9_t1_phasing_acad11": [
3,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
3,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
1,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
3,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
2,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
3,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
1,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
1,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
1,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
1,
3
]
}
},
{
"id": "grok-4-3-mini-swe-agent",
"modelName": "grok-4.3",
"modelLabel": "Grok 4.3",
"provider": "xAI",
"harness": "mini-swe-agent",
"harnessLabel": "mini-SWE",
"config": "Grok 4.3 / mini-SWE",
"passRate": 20.3,
"ci": [
14,
26.6
],
"nEvals": 118,
"correct": 72,
"gradeable": 354,
"runs": 354,
"avgCost": 0.14,
"totalCost": 49.01,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
3,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
0,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
0,
3
],
"CNV_04": [
3,
3
],
"CNV_05": [
3,
3
],
"CNV_07": [
0,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
1,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
0,
3
],
"E3_detection_denominator_donor_availability": [
0,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
3,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
1,
3
],
"gene_family_dosage_single": [
2,
3
],
"genetic_correlation_meta_eur_traits": [
1,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
1,
3
],
"GWAS_12": [
0,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
1,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
2,
3
],
"h20_context_sample_size_not_biology": [
1,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
0,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
0,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
0,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
2,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
1,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
0,
3
],
"mt_diploid_allele_count": [
0,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
3,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
0,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
0,
3
],
"n070_meta_betas_log_vs_linear": [
0,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
0,
3
],
"plasma_lactate_hypothesis_gen_blind": [
0,
3
],
"plasma_lactate_variant_id": [
0,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
1,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
0,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
0,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
0,
3
],
"pmid26771485_size_window_03": [
2,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
0,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
1,
3
],
"RFC1_01": [
0,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
1,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
3,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
2,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
2,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
0,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
0,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
1,
3
],
"variant_cohort_region_depth_duplication_carriers": [
1,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
2,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
2,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
3,
3
]
}
},
{
"id": "grok-4-3-pi",
"modelName": "grok-4.3",
"modelLabel": "Grok 4.3",
"provider": "xAI",
"harness": "pi",
"harnessLabel": "PI",
"config": "Grok 4.3 / PI",
"passRate": 19.5,
"ci": [
13.7,
25.3
],
"nEvals": 118,
"correct": 69,
"gradeable": 354,
"runs": 354,
"avgCost": 0.03,
"totalCost": 9.59,
"evalBreakdown": {
"A4_rsid_merge_remap": [
0,
3
],
"alignment_and_preprocessing_pedigree_relatedness_verification": [
2,
3
],
"annotation_and_interpretation_pathway_drug_target_triage": [
0,
3
],
"annotation_and_interpretation_pgx_star_allele_phasing": [
0,
3
],
"cis_mr_bcaa": [
0,
3
],
"cis_mr_ld_confound": [
0,
3
],
"cis_mr_opposite": [
0,
3
],
"CNV_04": [
1,
3
],
"CNV_05": [
2,
3
],
"CNV_07": [
1,
3
],
"CNV_08": [
3,
3
],
"CNV_12": [
3,
3
],
"coloc_hmgcr": [
0,
3
],
"coloc_hmgcr_splice": [
0,
3
],
"coloc_ldl_liver_data_quality": [
0,
3
],
"DR_01_t2d_polygenic_score": [
0,
3
],
"E3_detection_denominator_donor_availability": [
1,
3
],
"E8_cryptic_relatedness_effective_n": [
0,
3
],
"eqtl_cis_window_build_mismatch": [
0,
3
],
"eqtl_context_effect_size_attenuation_not_portability": [
2,
3
],
"eqtl_locus_robustness_cross_mappability_filter": [
0,
3
],
"eqtl_trans_target_count_multiple_testing": [
0,
3
],
"fine_mapping_effector_gene_prioritization": [
1,
3
],
"gene_family_dosage_single": [
1,
3
],
"genetic_correlation_meta_eur_traits": [
0,
3
],
"glucose_finemap_ld_mismatch": [
0,
3
],
"gwas_02_independent_signals_vs_distinct_loci": [
0,
3
],
"GWAS_02_strongest_locus_effect_vs_significance": [
0,
3
],
"GWAS_08": [
0,
3
],
"GWAS_12": [
1,
3
],
"gwas_24_gene_mapping_prioritization_signal_density": [
0,
3
],
"gwas_25_causal_variant_fine_mapping_resolution": [
2,
3
],
"gwas_28_prs_prediction_accuracy_validation": [
3,
3
],
"gwas_30_cross_ancestry_heritability_reference_mismatch": [
0,
3
],
"gwas_33_acan_supported_variant_sets": [
0,
3
],
"gwas_36_skeletal_growth_regulatory_genes": [
0,
3
],
"gwas_association_module_vs_individual_robustness": [
2,
3
],
"gwas_chry_nonpar_female_phantom_carriers": [
0,
3
],
"h20_context_sample_size_not_biology": [
2,
3
],
"h20_trans_targets_not_effector_gene": [
0,
3
],
"heritability_hdl_size_ldsc_estimator_meta_eur": [
0,
3
],
"heritability_hdl_size_meta_eur": [
0,
3
],
"heritability_hdl_size_pvalue_meta_eur": [
0,
3
],
"heterogeneity_model_choice_bcaa_chr4": [
3,
3
],
"humanvar_07_population_scale_function_blindspot": [
1,
3
],
"ld_reference_eur_dedup": [
0,
3
],
"male_chrx_nonpar_false_het": [
1,
3
],
"mendelian_randomization_trans_qtl_bystander_vs_causal": [
1,
3
],
"missing_homref": [
0,
3
],
"mouse_annotation_association_lineage_confounded_svs": [
1,
3
],
"mouse_annotation_vep_lof_count": [
0,
3
],
"mouse_filtering_inbred_singleton_indel_count": [
0,
3
],
"mouse_filtering_mgp_indel_discordant_calls_shared_loci": [
1,
3
],
"mouse_variant_calling_inbred_heterozygous_call_reliability": [
3,
3
],
"mr_glucose_t2d_genomewide": [
1,
3
],
"mt_diploid_allele_count": [
2,
3
],
"multiomics_mendelian_incidental_carrier_vs_affected": [
3,
3
],
"multiomics_phased_compound_het_biallelic_knockout": [
0,
3
],
"multiomics_rna_editing_genuine_vs_snp_artifact": [
0,
3
],
"n070_meta_betas_log_vs_linear": [
0,
3
],
"ont_rfc1_repeat_expansion_pathogenicity": [
0,
3
],
"ont_xylt1_repeat_expansion_transmission_risk": [
0,
3
],
"PGX01_conservative_validation_pgx_summary": [
0,
3
],
"plasma_lactate_causal": [
1,
3
],
"plasma_lactate_hypothesis_gen_blind": [
0,
3
],
"plasma_lactate_variant_id": [
0,
3
],
"pmid26547235_batch_effect_dna_repair_14": [
2,
3
],
"pmid26547235_callable_region_denominator_04": [
0,
3
],
"pmid26547235_chry_father_son_concordance_11": [
0,
3
],
"pmid26547235_chry_hemizygous_male_10": [
0,
3
],
"pmid26547235_family_denovo_snv_01": [
0,
3
],
"pmid26547235_female_chry_absence_12_hard": [
0,
3
],
"pmid26547235_microdeletion_survey_het_02": [
0,
3
],
"pmid26547235_mnp_blind_spot_09": [
1,
3
],
"pmid26547235_mother_chrx_son_transmission_13": [
1,
3
],
"pmid26547235_recessive_vs_denovo_05": [
0,
3
],
"pmid26771485_motif_norm_05": [
0,
3
],
"pmid26771485_size_window_03": [
3,
3
],
"pmid40670386_ont_systematic_errors_07": [
0,
3
],
"pmid40670386_strand_bias_01": [
2,
3
],
"Q1b_vntr_contraction_chr19": [
0,
3
],
"Q2_D2_amy1_mappability": [
0,
3
],
"Q3_D3_chm13_divergence_snps_chr19": [
0,
3
],
"q9_t1_phasing_acad11": [
0,
3
],
"qc_04_chr16_segdup_common_snp_count": [
0,
3
],
"QC_05_dual_indel_consensus": [
1,
3
],
"RFC1_01": [
2,
3
],
"rnaseq_aberrant_splicing_mendelian_diagnosis_fibroblast": [
0,
3
],
"rnaseq_multimodal_evidence_integration_candidate_prioritization_fibroblast": [
0,
3
],
"smaht_long_read_indel_filter_06": [
0,
3
],
"small_variant_caller_analysis": [
0,
3
],
"small_variant_calling_mnv_codon_consequence": [
0,
3
],
"small_variant_error_interp": [
0,
3
],
"small_variant_inheritance": [
0,
3
],
"somatic_sv_derivation_force_genotype_vs_subtract": [
0,
3
],
"specialized_germline_analyses_hla_mhc_call_reliability": [
0,
3
],
"specialized_germline_analyses_mtdna_heteroplasmy_feasibility": [
2,
3
],
"sv_chromosome_end_signal": [
0,
3
],
"sv_population_fst_top_differentiated_chr20": [
0,
3
],
"sv_size_peak_alu": [
0,
3
],
"sv_size_peak_line": [
0,
3
],
"tr_inheritance_consistency": [
0,
3
],
"tr_structure_proportions": [
3,
3
],
"variant_01_trio_denovo_parental_confirmation": [
0,
3
],
"variant_02_trio_mendelian_error_rate_missing_handling": [
1,
3
],
"variant_03_trio_sv_denovo_parental_confirmation": [
0,
3
],
"variant_04_twin_genome_concordance_artifact": [
0,
3
],
"variant_06_homozygous_nonparental_allele_artifact": [
1,
3
],
"variant_07_structural_variant_size_threshold_overcount": [
0,
3
],
"variant_08_per_chromosome_igh_somatic_inheritance_artifact": [
0,
3
],
"variant_cohort_region_depth_duplication_carriers": [
0,
3
],
"variant_filtering_and_benchmarking_child_homozygous_nonparental": [
0,
3
],
"variant_filtering_and_benchmarking_lof_artifact_triage": [
0,
3
],
"variantbench_mmej_deletion_fraction": [
0,
3
],
"variantbench_multiplatform_focal_amplification": [
0,
3
],
"variantbench_sv_platform_reconciliation": [
3,
3
],
"WES_C_gc_confounded_heterozygosity": [
0,
3
],
"WG_01_reference_absent_overclaim": [
2,
3
]
}
}
],
"examples": [
{
"id": "cis_mr_bcaa",
"title": "Cis Mr Bcaa",
"description": "Your lab is investigating whether lowering plasma BCAA levels via inhibition of BCKDK kinase could reduce T2D risk.",
"platform": "Population & Statistical Genomics",
"category": "Benchmark",
"passRate": 0,
"prompt": "Your lab is investigating whether lowering plasma BCAA levels via inhibition of BCKDK kinase could reduce T2D risk.\n\nIn order to do so, you have obtained GWAS summary statistics on T2D risk, as well as circulating BCAA levels, all in European ancestry populations.\n\nAs the degree of genetic correlation between circulating metabolites and heterogeneity between variants suggests a high degree of horizontal pleiotropy and difficulty in interpretation, you wish to avoid these issues while modeling the predicted change in T2D risk per 1 SD increase in levels of circulating BCAA levels at 3 best selected loci.\n\nWhat is the highest change you observe and the lowest across the loci that you investigate? Does genetic evidence support the idea that BCKDK inhibition would be beneficial for reducing T2D risk (answer \"YES\" or \"NO\") in a list?\n\nReturn EXACTLY:\n\n\n{\"loci_gene_names\": [], \"highest_estimate\": \"\", \"lowest_estimate\": \"\", \"bckdk_inhibition_yn\": []}\n",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/cis_mr_bcaa/eval.json",
"graderType": "label_set_jaccard + numeric_tolerance + label_set_jaccard",
"runs": [
{
"runId": "openai__gpt-5.6-sol__openai-codex__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "openai-codex",
"run": "r1",
"passed": true,
"score": 1,
"reasoning": "",
"metrics": {
"score": 1
},
"fieldScores": {},
"agentAnswer": {
"bckdk_inhibition_yn": [
"NO"
],
"highest_estimate": "0.110026",
"loci_gene_names": [
"BCAT2",
"DBT",
"PPM1K"
],
"lowest_estimate": "-0.052155"
},
"durationS": 624.623826,
"cost": null,
"nTurns": 1,
"trajectoryFile": "variantbench/trajectories/cis_mr_bcaa__openai__gpt-5.6-sol__openai-codex__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0.3333333333333333,
"reasoning": "",
"metrics": {
"score": 0.3333333333333333
},
"fieldScores": {},
"agentAnswer": {
"bckdk_inhibition_yn": [
"YES",
"NO",
"YES"
],
"highest_estimate": "PPM1K OR = 1.12 for T2D per 1-SD higher circulating BCAA (95% CI 1.03-1.22; log-OR +0.115; p=0.008)",
"loci_gene_names": [
"PPM1K",
"BCAT2",
"DBT"
],
"lowest_estimate": "BCAT2 OR = 0.98 for T2D per 1-SD higher circulating BCAA (95% CI 0.80-1.19; log-OR -0.024; p=0.81)"
},
"durationS": 2091.066878,
"cost": 4.8948905,
"nTurns": 43,
"trajectoryFile": "variantbench/trajectories/cis_mr_bcaa__anthropic__claude-opus-4-8__pi__r1.json"
},
{
"runId": "xai__grok-4.5__mini-swe-agent__r1",
"provider": "xAI",
"model": "grok-4.5",
"harness": "mini-swe-agent",
"run": "r1",
"passed": false,
"score": 0.6666666666666666,
"reasoning": "",
"metrics": {
"score": 0.6666666666666666
},
"fieldScores": {},
"agentAnswer": {
"bckdk_inhibition_yn": [
"YES"
],
"highest_estimate": "0.1154",
"loci_gene_names": [
"PPM1K",
"BCAT2",
"DBT"
],
"lowest_estimate": "-0.0240"
},
"durationS": 2416.72274,
"cost": 1.39095,
"nTurns": 51,
"trajectoryFile": "variantbench/trajectories/cis_mr_bcaa__xai__grok-4.5__mini-swe-agent__r1.json"
},
{
"runId": "gemini__gemini-3.5-flash__pi__r1",
"provider": "Google",
"model": "gemini-3.5-flash",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0.16666666666666666,
"reasoning": "",
"metrics": {
"score": 0.16666666666666666
},
"fieldScores": {},
"agentAnswer": {
"bckdk_inhibition_yn": [
"YES"
],
"highest_estimate": "1.15",
"loci_gene_names": [
"PPM1K",
"BCKDK",
"DBT"
],
"lowest_estimate": "0.03"
},
"durationS": 2666.114829,
"cost": 2.4421089,
"nTurns": 75,
"trajectoryFile": "variantbench/trajectories/cis_mr_bcaa__gemini__gemini-3.5-flash__pi__r1.json"
}
]
},
{
"id": "fine_mapping_effector_gene_prioritization",
"title": "Fine Mapping Effector Gene Prioritization",
"description": "You are given three tables for a chr22 disease-associated locus: (1) the genes trans-regulated by the locus with their effect estimates; (2) the variants in the 95% credible set for the locus signal, each annotated with its functional consequence, the gene it falls in, and (where coding) SIFT and PolyPhen predictions; (3) the cis association of the gene phys",
"platform": "Population & Statistical Genomics",
"category": "Benchmark",
"passRate": 0,
"prompt": "You are given three tables for a chr22 disease-associated locus: (1) the genes trans-regulated by the locus with their effect estimates; (2) the variants in the 95% credible set for the locus signal, each annotated with its functional consequence, the gene it falls in, and (where coding) SIFT and PolyPhen predictions; (3) the cis association of the gene physically spanning the credible set with its own expression across the surrounding region.\n\nNominate the causal (effector) gene at this locus and the variant class most likely to mediate its effect.\n\nReport the number of protein-altering variants in the credible set, and your interpretation.\n\nReturn EXACTLY:\n\n\n{\"n_protein_altering_variants_in_credible_set\": , \"answer\": \"\"}\n\n\nwhere is one of A, B, C, D, E, F:\nA. The effector is the gene spanning the credible set, acting through the intronic lead variant's regulation of that gene's own expression (a cis-expression mechanism).\nB. The effector is the gene spanning the credible set, acting through a protein-altering (missense) variant in the credible set, even though that variant is computationally predicted benign and the gene has no detectable cis-eQTL.\nC. The effector is whichever gene at the locus has the strongest cis-eQTL, following standard locus-to-gene expression prioritisation.\nD. No coding causal mechanism is supported, because protein-altering variants in the credible set are predicted benign, so the effector must act non-coding.\nE. The effector is the gene physically closest to the lead variant, by proximity.\nF. The effector is the most strongly trans-associated gene, since it carries the strongest association at the locus.",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/fine_mapping_effector_gene_prioritization/eval.json",
"graderType": "numeric_tolerance + multiple_choice",
"runs": [
{
"runId": "openai__gpt-5.6-sol__openai-codex__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "openai-codex",
"run": "r1",
"passed": true,
"score": 1,
"reasoning": "",
"metrics": {
"score": 1
},
"fieldScores": {},
"agentAnswer": {
"answer": "B",
"n_protein_altering_variants_in_credible_set": 1
},
"durationS": 60.336873,
"cost": null,
"nTurns": 1,
"trajectoryFile": "variantbench/trajectories/fine_mapping_effector_gene_prioritization__openai__gpt-5.6-sol__openai-codex__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": true,
"score": 1,
"reasoning": "",
"metrics": {
"score": 1
},
"fieldScores": {},
"agentAnswer": {
"answer": "B",
"n_protein_altering_variants_in_credible_set": 1
},
"durationS": 178.282173,
"cost": 0.49538075,
"nTurns": 8,
"trajectoryFile": "variantbench/trajectories/fine_mapping_effector_gene_prioritization__anthropic__claude-opus-4-8__pi__r1.json"
},
{
"runId": "xai__grok-4.5__mini-swe-agent__r1",
"provider": "xAI",
"model": "grok-4.5",
"harness": "mini-swe-agent",
"run": "r1",
"passed": false,
"score": 0.5,
"reasoning": "",
"metrics": {
"score": 0.5
},
"fieldScores": {},
"agentAnswer": {
"answer": "A",
"n_protein_altering_variants_in_credible_set": 1
},
"durationS": 196.500348,
"cost": 0.177606,
"nTurns": 10,
"trajectoryFile": "variantbench/trajectories/fine_mapping_effector_gene_prioritization__xai__grok-4.5__mini-swe-agent__r1.json"
},
{
"runId": "gemini__gemini-3.5-flash__pi__r1",
"provider": "Google",
"model": "gemini-3.5-flash",
"harness": "pi",
"run": "r1",
"passed": true,
"score": 1,
"reasoning": "",
"metrics": {
"score": 1
},
"fieldScores": {},
"agentAnswer": {
"answer": "B",
"n_protein_altering_variants_in_credible_set": 1
},
"durationS": 130.352735,
"cost": 0.36835604999999993,
"nTurns": 20,
"trajectoryFile": "variantbench/trajectories/fine_mapping_effector_gene_prioritization__gemini__gemini-3.5-flash__pi__r1.json"
}
]
},
{
"id": "chry_father_son_concordance_11",
"title": "Chry Father Son Concordance 11",
"description": "This dataset is a joint-called family quartet whole-exome sequencing VCF (Illumina, GRCh37) with proband `SAMPLE_A` and his father `SAMPLE_B` among the four samples. Report the number of biallelic PASS SNV positions on non-PAR chromosome Y where father and son most likely have discordant genotypes.",
"platform": "Personal & Clinical Genomics",
"category": "Benchmark",
"passRate": 0,
"prompt": "This dataset is a joint-called family quartet whole-exome sequencing VCF (Illumina, GRCh37) with proband `SAMPLE_A` and his father `SAMPLE_B` among the four samples. Report the number of biallelic PASS SNV positions on non-PAR chromosome Y where father and son most likely have discordant genotypes.\n\nReport:\n\n - `n_chry_father_son_discordant`: integer count.\n\nReturn EXACTLY:\n\n\n{\"n_chry_father_son_discordant\": }\n",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/chry_father_son_concordance_11/eval.json",
"graderType": "numeric_tolerance",
"runs": [
{
"runId": "openai__gpt-5.6-sol__openai-codex__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "openai-codex",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"n_chry_father_son_discordant": 36
},
"durationS": 96.528746,
"cost": null,
"nTurns": 1,
"trajectoryFile": "variantbench/trajectories/chry_father_son_concordance_11__openai__gpt-5.6-sol__openai-codex__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"n_chry_father_son_discordant": 32
},
"durationS": 513.754493,
"cost": 1.5976635,
"nTurns": 23,
"trajectoryFile": "variantbench/trajectories/chry_father_son_concordance_11__anthropic__claude-opus-4-8__pi__r1.json"
},
{
"runId": "xai__grok-4.5__mini-swe-agent__r1",
"provider": "xAI",
"model": "grok-4.5",
"harness": "mini-swe-agent",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"n_chry_father_son_discordant": 36
},
"durationS": 165.259557,
"cost": 0.27513,
"nTurns": 18,
"trajectoryFile": "variantbench/trajectories/chry_father_son_concordance_11__xai__grok-4.5__mini-swe-agent__r1.json"
},
{
"runId": "gemini__gemini-3.5-flash__pi__r1",
"provider": "Google",
"model": "gemini-3.5-flash",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"n_chry_father_son_discordant": 36
},
"durationS": 253.145217,
"cost": 0.9675631500000001,
"nTurns": 39,
"trajectoryFile": "variantbench/trajectories/chry_father_son_concordance_11__gemini__gemini-3.5-flash__pi__r1.json"
}
]
},
{
"id": "ont_rfc1_repeat_expansion_pathogenicity",
"title": "Ont Rfc1 Repeat Expansion Pathogenicity",
"description": "This is a long-read (Oxford Nanopore) whole-genome sequencing dataset from a single human individual, aligned to the GRCh38 reference genome, provided as read alignments over tandem repeat regions together with a tandem-repeat region catalog.",
"platform": "Personal & Clinical Genomics",
"category": "Benchmark",
"passRate": 0,
"prompt": "This is a long-read (Oxford Nanopore) whole-genome sequencing dataset from a single human individual, aligned to the GRCh38 reference genome, provided as read alignments over tandem repeat regions together with a tandem-repeat region catalog.\n\nBased on this individual's genotype at the RFC1 locus (chr4:39,348,424, intron 2 of RFC1), determine whether they are affected by, or a carrier for, the RFC1-associated neurological disorder (CANVAS / RFC1-spectrum disease).\n\nReturn EXACTLY:\n\n\n{\"n_pathogenic_expansion_alleles\": , \"expanded_allele_g_fraction\": }\n\n\nwhere n_pathogenic_expansion_alleles is the number of RFC1 alleles in this individual that carry a pathogenic (disease-causing) repeat expansion, and expanded_allele_g_fraction is the fraction of guanine (G) bases in the repeat tract of the expanded (longer) RFC1 allele.",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/ont_rfc1_repeat_expansion_pathogenicity/eval.json",
"graderType": "numeric_tolerance",
"runs": [
{
"runId": "openai__gpt-5.6-sol__openai-codex__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "openai-codex",
"run": "r1",
"passed": false,
"score": 0.5,
"reasoning": "",
"metrics": {
"score": 0.5
},
"fieldScores": {},
"agentAnswer": {
"expanded_allele_g_fraction": 0.33,
"n_pathogenic_expansion_alleles": 0
},
"durationS": 3060.460065,
"cost": null,
"nTurns": 1,
"trajectoryFile": "variantbench/trajectories/ont_rfc1_repeat_expansion_pathogenicity__openai__gpt-5.6-sol__openai-codex__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "Numeric Tolerance Check: FAIL\n\n x expanded_allele_g_fraction: 0.55 vs 0.2 (error: 0.3500)\n x n_pathogenic_expansion_alleles: 1 vs 0 (error: 1.0000)\n\nFailures:\n - expanded_allele_g_fraction: 0.55 vs 0.2 (error: 0.35, tolerance: 0.1)\n - n_pathogenic_expansion_alleles: 1 vs 0 (error: 1.00, tolerance: 0)",
"metrics": {
"expanded_allele_g_fraction_actual": 0.55,
"expanded_allele_g_fraction_expected": 0.2,
"expanded_allele_g_fraction_error": 0.35000000000000003,
"expanded_allele_g_fraction_pass": false,
"n_pathogenic_expansion_alleles_actual": 1,
"n_pathogenic_expansion_alleles_expected": 0,
"n_pathogenic_expansion_alleles_error": 1,
"n_pathogenic_expansion_alleles_pass": false
},
"fieldScores": {
"expanded_allele_g_fraction": 0,
"n_pathogenic_expansion_alleles": 0
},
"agentAnswer": {
"expanded_allele_g_fraction": 0.55,
"n_pathogenic_expansion_alleles": 1
},
"durationS": 2721.849611,
"cost": 10.70578575,
"nTurns": 27,
"trajectoryFile": "variantbench/trajectories/ont_rfc1_repeat_expansion_pathogenicity__anthropic__claude-opus-4-8__pi__r1.json"
},
{
"runId": "xai__grok-4.5__mini-swe-agent__r1",
"provider": "xAI",
"model": "grok-4.5",
"harness": "mini-swe-agent",
"run": "r1",
"passed": true,
"score": 1,
"reasoning": "",
"metrics": {
"score": 1
},
"fieldScores": {},
"agentAnswer": {
"expanded_allele_g_fraction": 0.2,
"n_pathogenic_expansion_alleles": 0
},
"durationS": 281.107925,
"cost": 1.107504,
"nTurns": 27,
"trajectoryFile": "variantbench/trajectories/ont_rfc1_repeat_expansion_pathogenicity__xai__grok-4.5__mini-swe-agent__r1.json"
},
{
"runId": "gemini__gemini-3.5-flash__pi__r1",
"provider": "Google",
"model": "gemini-3.5-flash",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"expanded_allele_g_fraction": 0.6,
"n_pathogenic_expansion_alleles": 1
},
"durationS": 1144.974935,
"cost": 4.524858450000002,
"nTurns": 86,
"trajectoryFile": "variantbench/trajectories/ont_rfc1_repeat_expansion_pathogenicity__gemini__gemini-3.5-flash__pi__r1.json"
}
]
},
{
"id": "small_variant_calling_mnv_codon_consequence",
"title": "Small Variant Calling Mnv Codon Consequence",
"description": "You are given a germline small-variant callset for a human individual from short-read Illumina exome sequencing, the aligned reads, and the reference sequence.",
"platform": "Variant Calling & QC",
"category": "Benchmark",
"passRate": 0,
"prompt": "You are given a germline small-variant callset for a human individual from short-read Illumina exome sequencing, the aligned reads, and the reference sequence.\n\nSome coding variants fall close enough to a neighbouring coding variant that both changes land in the same codon; there the amino-acid consequence of the pair is not the sum of the two scored separately. Restrict this to loci where establishing the combined consequence genuinely requires resolving phase between the neighbouring variants -- loci where it is not already fixed which changes sit together on one haplotype. Report:\n- n_codon_shared_consequence_changes: the number of coding variants whose predicted amino-acid consequence changes once these phase-dependent same-codon neighbours are interpreted jointly, relative to annotating each variant independently.\n\nWork from the provided files; do not perform new alignment or assembly. You may draw on a standard gene model / transcript annotation resource to predict consequences, but do not otherwise consult external variant databases.\n\nReturn EXACTLY:\n\n\n{\"n_codon_shared_consequence_changes\": }\n",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/small_variant_calling_mnv_codon_consequence/eval.json",
"graderType": "numeric_tolerance",
"runs": [
{
"runId": "openai__gpt-5.6-sol__openai-codex__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "openai-codex",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"n_codon_shared_consequence_changes": 33
},
"durationS": 1156.049691,
"cost": null,
"nTurns": 1,
"trajectoryFile": "variantbench/trajectories/small_variant_calling_mnv_codon_consequence__openai__gpt-5.6-sol__openai-codex__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": true,
"score": 1,
"reasoning": "",
"metrics": {
"score": 1
},
"fieldScores": {},
"agentAnswer": {
"n_codon_shared_consequence_changes": 99
},
"durationS": 4001.675866,
"cost": 11.846847249999998,
"nTurns": 70,
"trajectoryFile": "variantbench/trajectories/small_variant_calling_mnv_codon_consequence__anthropic__claude-opus-4-8__pi__r1.json"
},
{
"runId": "xai__grok-4.5__mini-swe-agent__r1",
"provider": "xAI",
"model": "grok-4.5",
"harness": "mini-swe-agent",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "Numeric Tolerance Check: FAIL\n\n x n_codon_shared_consequence_changes: 106 vs 82 (error: 24.0000)\n\nFailures:\n - n_codon_shared_consequence_changes: 106 vs 82 (error: 24.00, tolerance: 22)",
"metrics": {
"n_codon_shared_consequence_changes_actual": 106,
"n_codon_shared_consequence_changes_expected": 82,
"n_codon_shared_consequence_changes_error": 24,
"n_codon_shared_consequence_changes_pass": false
},
"fieldScores": {
"n_codon_shared_consequence_changes": 0
},
"agentAnswer": {
"n_codon_shared_consequence_changes": 106
},
"durationS": 1433.081192,
"cost": 0.9307559999999999,
"nTurns": 31,
"trajectoryFile": "variantbench/trajectories/small_variant_calling_mnv_codon_consequence__xai__grok-4.5__mini-swe-agent__r1.json"
},
{
"runId": "gemini__gemini-3.5-flash__pi__r1",
"provider": "Google",
"model": "gemini-3.5-flash",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"n_codon_shared_consequence_changes": 111
},
"durationS": 1402.642098,
"cost": 1.4966283000000002,
"nTurns": 53,
"trajectoryFile": "variantbench/trajectories/small_variant_calling_mnv_codon_consequence__gemini__gemini-3.5-flash__pi__r1.json"
}
]
},
{
"id": "sv_chromosome_end_signal",
"title": "Sv Chromosome End Signal",
"description": "I'm going to performing some downstream analysis on a structural variant (SV) callset for a seven-member family (samples `LG005`–`LG011`). Please note that originally, there were 10 samples but only 7 consented to our downstream analysis; thus, I expect the callset to contain some inconsequential artifacts in that regard.",
"platform": "Variant Calling & QC",
"category": "Benchmark",
"passRate": 0,
"prompt": "I'm going to performing some downstream analysis on a structural variant (SV) callset for a seven-member family (samples `LG005`–`LG011`). Please note that originally, there were 10 samples but only 7 consented to our downstream analysis; thus, I expect the callset to contain some inconsequential artifacts in that regard.\n\nThe callset is aligned to GRCh38 and was built by merging SV calls across sequencing technologies and callers. We use standard INFO fields and per-sample genotypes. A colleague of mine built it a long time ago so we're not sure exactly how he did it.\n\nCan you help me to QC this callset? I want to understand whether I can use it for downstream analysis of structural variants. On hand I have the callset and the GRCh38 reference index. Please indicate whether the dataset is overall usable or not (\"YES\" or \"NO\"). If the entire dataset isn't usable, are there any chromosomes that DO seem to be usable? Indicate which ones these might be in the form `chrN`. If the dataset is valid, simply put the single entry \"VALID\".\n\nReturn EXACTLY in this form (a single JSON object):\n\n\n{\"no_issue_chromosomes_if_any\": [\"chrN\", ...], \"valid_yes_or_no\": [\"YES\", ...]}\n",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/sv_chromosome_end_signal/eval.json",
"graderType": "all_of",
"runs": [
{
"runId": "openai__gpt-5.6-sol__openai-codex__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "openai-codex",
"run": "r1",
"passed": false,
"score": 0.5,
"reasoning": "",
"metrics": {
"score": 0.5
},
"fieldScores": {},
"agentAnswer": {
"no_issue_chromosomes_if_any": [
"chr1",
"chr2",
"chr3",
"chr4",
"chr5",
"chr6",
"chr8",
"chr9",
"chr10",
"chr11",
"chr12",
"chr13",
"chr14",
"chr15",
"chr16",
"chr18",
"chr19",
"chr20",
"chr21",
"chr22",
"chrX"
],
"valid_yes_or_no": [
"NO"
]
},
"durationS": 658.804647,
"cost": null,
"nTurns": 1,
"trajectoryFile": "variantbench/trajectories/sv_chromosome_end_signal__openai__gpt-5.6-sol__openai-codex__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0.5,
"reasoning": "",
"metrics": {
"score": 0.5
},
"fieldScores": {},
"agentAnswer": {
"no_issue_chromosomes_if_any": [
"chr1",
"chr2",
"chr3",
"chr4",
"chr5",
"chr6",
"chr8",
"chr9",
"chr10",
"chr11",
"chr12",
"chr13",
"chr14",
"chr15",
"chr16",
"chr18",
"chr19",
"chr20",
"chr21",
"chr22",
"chrX"
],
"valid_yes_or_no": [
"NO"
]
},
"durationS": 709.670379,
"cost": 2.0980374999999998,
"nTurns": 23,
"trajectoryFile": "variantbench/trajectories/sv_chromosome_end_signal__anthropic__claude-opus-4-8__pi__r1.json"
},
{
"runId": "xai__grok-4.5__mini-swe-agent__r1",
"provider": "xAI",
"model": "grok-4.5",
"harness": "mini-swe-agent",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"no_issue_chromosomes_if_any": [
"VALID"
],
"valid_yes_or_no": [
"YES"
]
},
"durationS": 335.595818,
"cost": 1.5183999999999995,
"nTurns": 47,
"trajectoryFile": "variantbench/trajectories/sv_chromosome_end_signal__xai__grok-4.5__mini-swe-agent__r1.json"
},
{
"runId": "gemini__gemini-3.5-flash__pi__r1",
"provider": "Google",
"model": "gemini-3.5-flash",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"no_issue_chromosomes_if_any": [
"VALID"
],
"valid_yes_or_no": [
"YES"
]
},
"durationS": 109.976243,
"cost": 0.3813306,
"nTurns": 25,
"trajectoryFile": "variantbench/trajectories/sv_chromosome_end_signal__gemini__gemini-3.5-flash__pi__r1.json"
}
]
},
{
"id": "osteosarcoma_wes_stage3_called_vcf_neoantigen_gene_recall",
"title": "Osteosarcoma Wes Stage3 Called Vcf Neoantigen Gene Recall",
"description": "You are given the raw somatic variant callsets from a paired tumor / matched-normal whole-exome sequencing experiment of a human tumor (hg38): short somatic variant calls, structural-variant calls, and copy-number segment calls for the tumor. These callsets are UNFILTERED — no call-quality selection has been applied to them — and it is your responsibility to",
"platform": "Neoantigen Prediction",
"category": "Neoantigen subset",
"passRate": 0,
"prompt": "You are given the raw somatic variant callsets from a paired tumor / matched-normal whole-exome sequencing experiment of a human tumor (hg38): short somatic variant calls, structural-variant calls, and copy-number segment calls for the tumor. These callsets are UNFILTERED — no call-quality selection has been applied to them — and it is your responsibility to decide on and apply whatever variant filtering and selection you judge appropriate before drawing any conclusions. Starting from these callsets, determine the tumor's final set of predicted neoantigens and report the gene hosting each predicted neoepitope.\n\nReference data (genome, gene annotation / VEP cache, HLA typing, and neoantigen-prediction resources) and tumor bulk RNA-seq expression estimates for the same tumor (RSEM gene- and isoform-level quantification, keyed by Ensembl IDs) are provided alongside the callsets; any or all of them may be needed.\n\nDo not access the internet, other than to install packages or through API requests from installed bioinformatics libraries. You should make your own judgements based on the data provided; internet data is unreliable for this clinical case.\n\nReturn EXACTLY:\n\n\n{\"neoantigen_genes\": [\"GENE1\", \"GENE2\", ...]}\n",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/osteosarcoma_wes_stage3_called_vcf_neoantigen_gene_recall/eval.json",
"graderType": "marker_gene_precision_recall",
"runs": [
{
"runId": "openai__gpt-5.6-sol__pi__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"neoantigen_genes": [
"DYNC1H1",
"MAP2",
"ZNF436"
]
},
"durationS": 3974.575917,
"cost": 12.993228000000002,
"nTurns": 122,
"trajectoryFile": "variantbench/trajectories/osteosarcoma_wes_stage3_called_vcf_neoantigen_gene_recall__openai__gpt-5.6-sol__pi__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "",
"metrics": {
"score": 0
},
"fieldScores": {},
"agentAnswer": {
"neoantigen_genes": [
"DYNC1H1",
"MAP2",
"ZNF436"
]
},
"durationS": 4366.32266,
"cost": 9.88558275,
"nTurns": 98,
"trajectoryFile": "variantbench/trajectories/osteosarcoma_wes_stage3_called_vcf_neoantigen_gene_recall__anthropic__claude-opus-4-8__pi__r1.json"
}
]
},
{
"id": "eval_stage4_annotated_vcf",
"title": "Eval Stage4 Annotated Vcf",
"description": "You are given functionally annotated somatic variant calls for a human Illumina whole-exome sequencing (WES) tumor / matched-normal pair, aligned to GRCh38. The calls are provided as-is, with no curation or prioritization applied. Tumor bulk RNA-seq expression estimates for the same tumor (RSEM gene- and isoform-level quantification, keyed by Ensembl IDs) ar",
"platform": "Neoantigen Prediction",
"category": "Neoantigen subset",
"passRate": 0,
"prompt": "You are given functionally annotated somatic variant calls for a human Illumina whole-exome sequencing (WES) tumor / matched-normal pair, aligned to GRCh38. The calls are provided as-is, with no curation or prioritization applied. Tumor bulk RNA-seq expression estimates for the same tumor (RSEM gene- and isoform-level quantification, keyed by Ensembl IDs) are provided alongside the calls. Determine this tumor's final neoantigen set and report the gene that hosts each predicted neoepitope. Decide and apply whatever variant selection and prioritization are appropriate on your own.\n\nDo not access the internet, other than to install packages or through API requests from installed bioinformatics libraries. You should make your own judgements based on the data provided; internet data is unreliable for this clinical case.\n\nReturn EXACTLY:\n\n\n{\"neoantigen_genes\": [\"GENE1\", \"GENE2\", ...]}\n",
"githubUrl": "https://github.com/latchbio/variantbench/blob/main/evals/eval_stage4_annotated_vcf/eval.json",
"graderType": "marker_gene_precision_recall",
"runs": [
{
"runId": "openai__gpt-5.6-sol__pi__r1",
"provider": "OpenAI",
"model": "gpt-5.6-sol",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "Marker Gene Precision/Recall: FAIL\nAnswer field: neoantigen_genes\n\n + Precision@20: 0.600 (threshold: 0.100)\n x Recall@20: 0.857 (threshold: 1.000)\n\nTrue Positives (12):\n + ADGRF5\n + CUL9\n + DYNC1H1\n + EPG5\n + EXOC4\n + KDM3B\n + KIF1C\n + MYO15B\n + MYO9A\n + NR2F2\n + SLC25A12\n + TECPR1\n\nFalse Negatives (2):\n - BTD\n - MAP2\n\nFailure: Recall 0.857 < 1.000",
"metrics": {
"k": 20,
"precision_at_k": 0.6,
"recall_at_k": 0.8571428571428571,
"precision_threshold": 0.1,
"recall_threshold": 1,
"true_positives": [
"ADGRF5",
"CUL9",
"DYNC1H1",
"EPG5",
"EXOC4",
"KDM3B",
"KIF1C",
"MYO15B",
"MYO9A",
"NR2F2",
"SLC25A12",
"TECPR1"
],
"false_positives": [
"H1-2",
"IGBP1",
"KRT18",
"MORN1",
"SMARCAL1",
"VPS13B",
"VSIG4",
"ZNF436"
],
"false_negatives": [
"BTD",
"MAP2"
],
"num_true_positives": 12,
"num_false_positives": 8,
"num_false_negatives": 2,
"num_canonical_markers": 14,
"precision_pass": true,
"recall_pass": false,
"answer_field_used": "neoantigen_genes"
},
"fieldScores": {},
"agentAnswer": {
"neoantigen_genes": [
"ADGRF5",
"CUL9",
"DYNC1H1",
"EPG5",
"EXOC4",
"H1-2",
"IGBP1",
"KDM3B",
"KIF1C",
"KRT18",
"MORN1",
"MYO15B",
"MYO9A",
"NR2F2",
"SLC25A12",
"SMARCAL1",
"TECPR1",
"VPS13B",
"VSIG4",
"ZNF436"
]
},
"durationS": 2387.687054,
"cost": 9.202331249999999,
"nTurns": 91,
"trajectoryFile": "variantbench/trajectories/eval_stage4_annotated_vcf__openai__gpt-5.6-sol__pi__r1.json"
},
{
"runId": "anthropic__claude-opus-4-8__pi__r1",
"provider": "Anthropic",
"model": "claude-opus-4-8",
"harness": "pi",
"run": "r1",
"passed": false,
"score": 0,
"reasoning": "Marker Gene Precision/Recall: FAIL\nAnswer field: neoantigen_genes\n\n + Precision@3: 0.667 (threshold: 0.100)\n x Recall@3: 0.143 (threshold: 1.000)\n\nTrue Positives (2):\n + BTD\n + DYNC1H1\n\nFalse Negatives (12):\n - ADGRF5\n - CUL9\n - EPG5\n - EXOC4\n - KDM3B\n - KIF1C\n - MAP2\n - MYO15B\n - MYO9A\n - NR2F2\n - SLC25A12\n - TECPR1\n\nFailure: Recall 0.143 < 1.000",
"metrics": {
"k": 3,
"precision_at_k": 0.6666666666666666,
"recall_at_k": 0.14285714285714285,
"precision_threshold": 0.1,
"recall_threshold": 1,
"true_positives": [
"BTD",
"DYNC1H1"
],
"false_positives": [
"ZNF436"
],
"false_negatives": [
"ADGRF5",
"CUL9",
"EPG5",
"EXOC4",
"KDM3B",
"KIF1C",
"MAP2",
"MYO15B",
"MYO9A",
"NR2F2",
"SLC25A12",
"TECPR1"
],
"num_true_positives": 2,
"num_false_positives": 1,
"num_false_negatives": 12,
"num_canonical_markers": 14,
"precision_pass": true,
"recall_pass": false,
"answer_field_used": "neoantigen_genes"
},
"fieldScores": {},
"agentAnswer": {
"neoantigen_genes": [
"DYNC1H1",
"BTD",
"ZNF436"
]
},
"durationS": 2945.265395,
"cost": 10.265676500000001,
"nTurns": 84,
"trajectoryFile": "variantbench/trajectories/eval_stage4_annotated_vcf__anthropic__claude-opus-4-8__pi__r1.json"
}
]
}
]
}