As researchers are developing methods for detecting and accommo- dating novelties that will make AI agents more robust to unknown sudden changes in the “open worlds”, there is an increasing need for benchmark environments that allow for the systematic evalua- tions of the proposed AI techniques. We present “NovelGridworlds”, an OpenAI Gym environment framework for developing and eval- uating AI agents that can detect and adapt to unknown sudden novelties in their environments. Based on a rich taxonomy of nov- elties illustrated in 4 different tasks with 12 novelties in each, we propose evaluation metrics for evaluating both planning systems and learning systems that can handle novelty and illustrate the metrics with results from simulations of both types of AI agents.
@inproceedings{goeletal21ala,
title={NovelGridworlds: A Benchmark Environment for Detecting and Adapting to Novelties in Open Worlds},
author={Goel, Shivam and Tatiya, Gyan and Scheutz, Matthias and Sinapov, Jivko},
year={2021},
booktitle={AAMAS Workshop on on Adaptive Learning Agents (ALA)},
url={https://hrilab.tufts.edu/publications/goeletal21ala.pdf}
}