Datasets

@inproceedings{bartolo-etal-2021-improving, title = "Improving Question Answering Model Robustness with Synthetic Adversarial Data Generation", author = "Bartolo, Max and Thrush, Tristan and Jia, Robin and Riedel, Sebastian and Stenetorp, Pontus and Kiela, Douwe", booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing", month = nov, year = "2021", address = "Online and Punta Cana, Dominican Republic", publisher = "Association for Computational Linguistics", url = "https://aclanthology.org/2021.emnlp-main.696", doi = "10.18653/v1/2021.emnlp-main.696", pages = "8830--8848", abstract = "Despite recent progress, state-of-the-art question answering models remain vulnerable to a variety of adversarial attacks. While dynamic adversarial data collection, in which a human annotator tries to write examples that fool a model-in-the-loop, can improve model robustness, this process is expensive which limits the scale of the collected data. In this work, we are the first to use synthetic adversarial data generation to make question answering models more robust to human adversaries. We develop a data generation pipeline that selects source passages, identifies candidate answers, generates questions, then finally filters or re-labels them to improve quality. Using this approach, we amplify a smaller human-written adversarial dataset to a much larger set of synthetic question-answer pairs. By incorporating our synthetic data, we improve the state-of-the-art on the AdversarialQA dataset by 3.7F1 and improve model generalisation on nine of the twelve MRQA datasets. We further conduct a novel human-in-the-loop evaluation and show that our models are considerably more robust to new human-written adversarial examples: crowdworkers can fool our model only 8.8{\%} of the time on average, compared to 17.6{\%} for a model trained without synthetic data.", }

@misc{lewis2021paq, title={PAQ: 65 Million Probably-Asked Questions and What You Can Do With Them}, author={Patrick Lewis and Yuxiang Wu and Linqing Liu and Pasquale Minervini and Heinrich Küttler and Aleksandra Piktus and Pontus Stenetorp and Sebastian Riedel}, year={2021}, eprint={2102.07033}, archivePrefix={arXiv}, primaryClass={cs.CL} }

@article{doi:10.1162/tacl\_a\_00338, author = {Bartolo, Max and Roberts, Alastair and Welbl, Johannes and Riedel, Sebastian and Stenetorp, Pontus}, title = {Beat the AI: Investigating Adversarial Human Annotation for Reading Comprehension}, journal = {Transactions of the Association for Computational Linguistics}, volume = {8}, number = {}, pages = {662-678}, year = {2020}, doi = {10.1162/tacl_a_00338}, URL = { https://doi.org/10.1162/tacl_a_00338 }, eprint = { https://doi.org/10.1162/tacl_a_00338 }, abstract = { Innovations in annotation methodology have been a catalyst for Reading Comprehension (RC) datasets and models. One recent trend to challenge current RC models is to involve a model in the annotation process: Humans create questions adversarially, such that the model fails to answer them correctly. In this work we investigate this annotation methodology and apply it in three different settings, collecting a total of 36,000 samples with progressively stronger models in the annotation loop. This allows us to explore questions such as the reproducibility of the adversarial effect, transfer from data collected with varying model-in-the-loop strengths, and generalization to data collected without a model. We find that training on adversarially collected samples leads to strong generalization to non-adversarially collected datasets, yet with progressive performance deterioration with increasingly stronger models-in-the-loop. Furthermore, we find that stronger models can still learn from datasets collected with substantially weaker models-in-the-loop. When trained on data collected with a BiDAF model in the loop, RoBERTa achieves 39.9F1 on questions that it cannot answer when trained on SQuAD—only marginally lower than when trained on data collected using RoBERTa itself (41.0F1). } }

@inproceedings{petroni2020kilt, title={KILT: a Benchmark for Knowledge Intensive Language Tasks}, author={Fabio Petroni, Aleksandra Piktus, Angela Fan, Patrick Lewis, Majid Yazdani, Nicola De Cao, James Thorne, Yacine Jernite, Vassilis Plachouras, Tim Rockt{\"{a}}schel, Sebastian Riedel}, booktitle={arXiv:2009.02252}, year={2020} }

@article{lewis2019mlqa, title={MLQA: Evaluating Cross-lingual Extractive Question Answering}, author={Lewis, Patrick and O\u{g}uz, Barlas and Rinott, Ruty and Riedel, Sebastian and Schwenk, Holger}, journal={arXiv preprint arXiv:1910.07475}, year={2019} }

@article{DBLP:journals/corr/abs-1809-01494, author = {Marzieh Saeidi and Max Bartolo and Patrick Lewis and Sameer Singh and Tim Rockt{\"{a}}schel and Mike Sheldon and Guillaume Bouchard and Sebastian Riedel}, title = {Interpretation of Natural Language Rules in Conversational Machine Reading}, journal = {CoRR}, volume = {abs/1809.01494}, year = {2018}, url = {http://arxiv.org/abs/1809.01494}, archivePrefix = {arXiv}, eprint = {1809.01494}, timestamp = {Fri, 05 Oct 2018 11:34:52 +0200}, biburl = {https://dblp.org/rec/bib/journals/corr/abs-1809-01494}, bibsource = {dblp computer science bibliography, https://dblp.org} }

@article{welbl2018tacl, volume={6}, journal={Transactions of the Association for Computational Linguistics}, pages={287--302}, keywords={}, issn={2307-387X}, year={2018}, author={Johannes Welbl, Pontus Stenetorp, Sebastian Riedel}, title={Constructing Datasets for Multi-hop Reading Comprehension Across Documents}, }