Measurement Science for Language Model Evaluation

@misc{salaudeen2025meaning,
  title = {{Measurement to Meaning: A Validity-Centered Framework for AI Evaluation}},
  author = {Salaudeen, Olawale and Reuel, Anka and Ahmed, Ahmed and Bedi, Suhana and Robertson, Zachary and Sundar, Sudharsan and Domingue, Ben and Wang, Angelina and Koyejo, Sanmi},
  year = {2025},
  howpublished = {arXiv preprint},
  url = {https://arxiv.org/abs/2505.10573}
}

@article{cronbach1955construct,
  title = {{Construct Validity in Psychological Tests}},
  author = {Cronbach, Lee J. and Meehl, Paul E.},
  year = {1955},
  journal = {Psychological Bulletin},
  doi = {10.1037/h0040957},
  url = {https://www.yorku.ca/~pclassic/Cronbach/construct.htm}
}

@article{campbell1959validation,
  title = {{Convergent and Discriminant Validation by the Multitrait-Multimethod Matrix}},
  author = {Campbell, Donald T. and Fiske, Donald W.},
  year = {1959},
  journal = {Psychological Bulletin},
  doi = {10.1037/h0046016},
  url = {https://doi.org/10.1037/h0046016}
}

@inproceedings{desai2026benchmarks,
  title = {{What AI Benchmarks Actually Measure: Adapting Convergent and Discriminant Validity to Interrogate Fifty-Six AI Benchmarks}},
  author = {Desai, Meera and Truong, Sang T. and Wallach, Hanna and Chouldechova, Alex and Cooper, A. Feder and Garcia-Gathright, Jean and Ho, Daniel E. and Jacobs, Abigail Z. and Koyejo, Sanmi and Pangakis, Nicholas and Wang, Angelina},
  year = {2026},
  booktitle = {Conference on Language Modeling},
  url = {https://arxiv.org/abs/2609.08812}
}

@inproceedings{ren2024safetywashing,
  title = {{Safetywashing: Do AI Safety Benchmarks Actually Measure Safety Progress?}},
  author = {Ren, Richard and Basart, Steven and Khoja, Adam and Gatti, Alice and Phan, Long and Yin, Xuwang and Mazeika, Mantas and Pan, Alexander and Mukobi, Gabriel and Kim, Ryan H. and Fitz, Stephen and Hendrycks, Dan},
  year = {2024},
  booktitle = {Advances in Neural Information Processing Systems},
  url = {https://arxiv.org/abs/2407.21792}
}

@article{bedi2026medhelm,
  title = {{Holistic Evaluation of Large Language Models for Medical Tasks with MedHELM}},
  author = {Bedi, Suhana and Cui, Hejie and Fuentes, Miguel and Unell, Alyssa and others},
  year = {2026},
  journal = {Nature Medicine},
  doi = {10.1038/s41591-025-04151-2},
  url = {https://www.nature.com/articles/s41591-025-04151-2}
}

@inproceedings{blodgett2021stereotyping,
  title = {{Stereotyping Norwegian Salmon: An Inventory of Pitfalls in Fairness Benchmark Datasets}},
  author = {Blodgett, Su Lin and Lopez, Gilsinia and Olteanu, Alexandra and Sim, Robert and Wallach, Hanna},
  year = {2021},
  booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
  doi = {10.18653/v1/2021.acl-long.81},
  url = {https://aclanthology.org/2021.acl-long.81/}
}

@inproceedings{truong2025amortized,
  title = {{Reliable and Efficient Amortized Model-based Evaluation}},
  author = {Truong, Sang T. and Tu, Yuheng and Liang, Percy and Li, Bo and Koyejo, Sanmi},
  year = {2025},
  booktitle = {Proceedings of the 42nd International Conference on Machine Learning},
  url = {https://proceedings.mlr.press/v267/truong25c.html}
}

@inproceedings{truong2025bugs,
  title = {{Fantastic Bugs and Where to Find Them in AI Benchmarks}},
  author = {Truong, Sang and Tu, Yuheng and Hardy, Michael and Reuel-Lamparth, Anka and Tang, Zeyu and Burapacheep, Jirayu and Perera, Jonathan and Uwakwe, Chibuike and Domingue, Benjamin and Haber, Nick and Koyejo, Sanmi},
  year = {2025},
  booktitle = {Advances in Neural Information Processing Systems},
  url = {https://papers.nips.cc/paper_files/paper/2025/hash/5b44b99d012922011f0be1031a98c4f9-Abstract-Datasets_and_Benchmarks_Track.html}
}

@inproceedings{schaeffer2023emergence,
  title = {{Are Emergent Abilities of Large Language Models a Mirage?}},
  author = {Schaeffer, Rylan and Miranda, Brando and Koyejo, Sanmi},
  year = {2023},
  booktitle = {Advances in Neural Information Processing Systems},
  url = {https://arxiv.org/abs/2304.15004}
}

@misc{karim2025translation,
  title = {{Lost in Cultural Translation: Do LLMs Struggle with Math Across Cultural Contexts?}},
  author = {Karim, Aabid and Karim, Abdul and Lohana, Bhoomika and Keon, Matt and Singh, Jaswinder and Sattar, Abdul},
  year = {2025},
  howpublished = {arXiv preprint · revised 2026},
  url = {https://arxiv.org/abs/2503.18018}
}

@inproceedings{singh2025globalmmlu,
  title = {{Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation}},
  author = {Singh, Shivalika and Romanou, Angelika and Fourrier, Clémentine and Adelani, David Ifeoluwa and others},
  year = {2025},
  booktitle = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  doi = {10.18653/v1/2025.acl-long.919},
  url = {https://aclanthology.org/2025.acl-long.919/}
}

@inproceedings{truong2024vietnamese,
  title = {{Crossing Linguistic Horizons: Finetuning and Comprehensive Evaluation of Vietnamese Large Language Models}},
  author = {Truong, Sang T. and Nguyen, Duc Quang and Nguyen, Toan Q. and Le, Dong D. and Truong, Nhi N. and Quan, Tho and Koyejo, Sanmi},
  year = {2024},
  booktitle = {Findings of the Association for Computational Linguistics: NAACL 2024},
  doi = {10.18653/v1/2024.findings-naacl.182},
  url = {https://aclanthology.org/2024.findings-naacl.182/}
}

@inproceedings{zhang2026guardrails,
  title = {{Why Do Safety Guardrails Degrade Across Languages?}},
  author = {Zhang, Max and Patel, Ameen and Truong, Sang T. and Koyejo, Sanmi},
  year = {2026},
  booktitle = {Conference on Language Modeling},
  url = {https://arxiv.org/abs/2605.17173}
}

@misc{metr2026maintainers,
  title = {{Many SWE-bench-Passing PRs Would Not Be Merged into Main}},
  author = {Whitfill, Parker and Wu, Cheryl and Becker, Joel and Rush, Nate},
  year = {2026},
  howpublished = {METR research note},
  url = {https://metr.org/notes/2026-03-10-many-swe-bench-passing-prs-would-not-be-merged-into-main/}
}

@inproceedings{recht2019imagenet,
  title = {{Do ImageNet Classifiers Generalize to ImageNet?}},
  author = {Recht, Benjamin and Roelofs, Rebecca and Schmidt, Ludwig and Shankar, Vaishaal},
  year = {2019},
  booktitle = {Proceedings of the 36th International Conference on Machine Learning},
  url = {https://proceedings.mlr.press/v97/recht19a.html}
}

@misc{salaudeen2024imagenot,
  title = {{ImageNot: A Contrast with ImageNet Preserves Model Rankings}},
  author = {Salaudeen, Olawale and Hardt, Moritz},
  year = {2024},
  howpublished = {arXiv preprint · revised 2025},
  url = {https://arxiv.org/abs/2404.02112}
}

@inproceedings{singh2025leaderboard,
  title = {{The Leaderboard Illusion}},
  author = {Singh, Shivalika and Nan, Yiyang and Wang, Alex and Dsouza, Daniel and Kapoor, Sayash and Üstün, Ahmet and Koyejo, Sanmi and Deng, Yuntian and Longpre, Shayne and Smith, Noah and Ermis, Beyza and Fadaee, Marzieh and Hooker, Sara},
  year = {2025},
  booktitle = {Advances in Neural Information Processing Systems},
  url = {https://papers.neurips.cc/paper_files/paper/2025/hash/70a93f260a51123b3c0e33ecd1b4de97-Abstract-Datasets_and_Benchmarks_Track.html}
}

@misc{jiang2024contamination,
  title = {{Investigating Data Contamination for Pre-training Language Models}},
  author = {Jiang, Minhao and Liu, Ken Ziyu and Zhong, Ming and Schaeffer, Rylan and Ouyang, Siru and Han, Jiawei and Koyejo, Sanmi},
  year = {2024},
  howpublished = {arXiv preprint},
  url = {https://arxiv.org/abs/2401.06059}
}

@inproceedings{schaeffer2025paradox,
  title = {{The Contamination Paradox: Why Test Set Leakage Can Be Both Potent and Negligible}},
  author = {Schaeffer, Rylan and Liu, Ken Ziyu and Miranda, Brando and Ahmed, Ahmed and Mireshghallah, Niloofar and Koyejo, Sanmi},
  year = {2025},
  booktitle = {NeurIPS Workshop on Evaluating the Evolving LLM Lifecycle},
  url = {https://openreview.net/forum?id=Ajr8YtfhVO}
}

@misc{schaeffer2026contamination,
  title = {{Quantifying the Effect of Test Set Contamination on Generative Evaluations}},
  author = {Schaeffer, Rylan and Kazdan, Joshua and Abbasi, Baber and Liu, Ken Ziyu and Miranda, Brando and Ahmed, Ahmed and Berez, Fazl and Puri, Abhay and Biderman, Stella and Mireshghallah, Niloofar and Koyejo, Sanmi},
  year = {2026},
  howpublished = {arXiv preprint},
  url = {https://arxiv.org/abs/2601.04301}
}

@inproceedings{akhtar2026plateau,
  title = {{When AI Benchmarks Plateau: A Systematic Study of Benchmark Saturation}},
  author = {Akhtar, Mubashara and Reuel, Anka and Soni, Prajna and Ahuja, Sanchit and others},
  year = {2026},
  booktitle = {Proceedings of the 43rd International Conference on Machine Learning},
  url = {https://arxiv.org/abs/2602.16763}
}

@misc{truong2026strategic,
  title = {{Strategic Evaluation: Incentivizing AI Capability Coverage with Private Benchmarks}},
  author = {Truong, Sang and Wang, Serena and Haber, Nick and Koyejo, Sanmi},
  year = {2026},
  howpublished = {Preprint},
  url = {https://aimslab.stanford.edu/papers/Truong2026StrategicEvaluation.pdf}
}

@inproceedings{blum2015ladder,
  title = {{The Ladder: A Reliable Leaderboard for Machine Learning Competitions}},
  author = {Blum, Avrim and Hardt, Moritz},
  year = {2015},
  booktitle = {Proceedings of the 32nd International Conference on Machine Learning},
  url = {https://proceedings.mlr.press/v37/blum15.html}
}

@inproceedings{dwork2015adaptive,
  title = {{Preserving Statistical Validity in Adaptive Data Analysis}},
  author = {Dwork, Cynthia and Feldman, Vitaly and Hardt, Moritz and Pitassi, Toniann and Reingold, Omer and Roth, Aaron},
  year = {2015},
  booktitle = {Proceedings of the 47th Annual ACM Symposium on Theory of Computing},
  url = {https://arxiv.org/abs/1411.2664}
}

@inproceedings{roelofs2019overfitting,
  title = {{A Meta-Analysis of Overfitting in Machine Learning}},
  author = {Roelofs, Rebecca and Shankar, Vaishaal and Recht, Benjamin and Fridovich-Keil, Sara and Hardt, Moritz and Miller, John and Schmidt, Ludwig},
  year = {2019},
  booktitle = {Advances in Neural Information Processing Systems},
  url = {https://papers.nips.cc/paper_files/paper/2019/hash/ee39e503b6bedf0c98c388b7e8589aca-Abstract.html}
}