The JHU CoSTAR Block Stacking Dataset includes a real robot trying to stack 5.1 cm colored blocks to complete an order-fulfillment style block stacking task. It contains dynamic scenes and real time-series data in a less constrained environment than comparable datasets. There are nearly 12,000 stacking attempts and over 2 million frames of real data.
@article{hundt2019costar,
author = {Andrew Hundt and Varun Jain and Chia-Hung Lin and Chris Paxton and Gregory D. Hager},
journal = {Intelligent Robots and Systems (IROS), 2019 IEEE International Conference on},
title = {The CoSTAR Block Stacking Dataset: Learning with Workspace Constraints},
url = {https://arxiv.org/abs/1810.11714},
year = {2019}
}