@inproceedings{Murray2024, title = {Learning to Grasp in Clutter with Interactive Visual Failure Prediction}, author = {Michael Murray, Abhishek Gupta, Maya Cakmak}, year = {2024}, abstract = {Modern warehouses process millions of unique objects which are often stored in densely packed containers. To automate tasks in this environment, a robot must be able to pick diverse objects from highly cluttered scenes. Real-world learning is a promising approach, but executing picks in the real world is time-consuming, can induce costly failures, and often requires extensive human intervention, which causes operational burden and limits the scope of data collection and deployments. In this work, we leverage interactive probes to visually evaluate grasps in clutter without fully executing picks, a capability we refer to as Interactive Visual Failure Prediction (IVFP). This enables autonomous verification of grasps during execution to avoid costly downstream failures as well as autonomous reward assignment, providing supervision to continuously shape and improve grasping behavior as the robot gathers experience in the real world, without constantly requiring human intervention. Through experiments on a Stretch RE1 robot, we study the effect that IVFP has on performance - both in terms of effective data throughput and success rate, and show that this approach leads to grasping policies that outperform policies trained with human supervision alone, while requiring significantly less human intervention.}, booktitle = {International Conference on Robotics and Automation (ICRA)} }

Abstract: Modern warehouses process millions of unique objects which are often stored in densely packed containers. To automate tasks in this environment, a robot must be able to pick diverse objects from highly cluttered scenes. Real-world learning is a promising approach, but executing picks in the real world is time-consuming, can induce costly failures, and often requires extensive human intervention, which causes operational burden and limits the scope of data collection and deployments. In this work, we leverage interactive probes to visually evaluate grasps in clutter without fully executing picks, a capability we refer to as Interactive Visual Failure Prediction (IVFP). This enables autonomous verification of grasps during execution to avoid costly downstream failures as well as autonomous reward assignment, providing supervision to continuously shape and improve grasping behavior as the robot gathers experience in the real world, without constantly requiring human intervention. Through experiments on a Stretch RE1 robot, we study the effect that IVFP has on performance - both in terms of effective data throughput and success rate, and show that this approach leads to grasping policies that outperform policies trained with human supervision alone, while requiring significantly less human intervention.

@inproceedings{Grotz2024bimanual, title = {PerAct2: Benchmarking and Learning for Robotic Bimanual Manipulation Tasks}, author = {Markus Grotz, Mohit Shridhar, Tamim Asfour, Dieter Fox}, year = {2024}, url = {https://bimanual.github.io}, file = {https://arxiv.org/pdf/2407.00278}, booktitle = {In preparation}, abstract = {Bimanual manipulation is challenging due to precise spatial and temporal coordination required between two arms. While there exist several real-world bimanual systems, there is a lack of simulated benchmarks with a large task diversity for systematically studying bimanual capabilities across a wide range of table- top tasks. This paper addresses the gap by extending RLBench to bimanual manipulation. We open-source our code and benchmark, which comprises 13 new tasks with 23 unique task variations, each requiring a high degree of coordination and adaptability. To kickstart the benchmark, we extended several state-of-the-art methods to bimanual manipulation and also present a language-conditioned behavioral cloning agent – PerAct2, an extension of the PerAct framework. This method enables the learning and execution of bimanual 6-DoF manipulation tasks. Our novel network architecture efficiently integrates language processing with action prediction, allowing robots to understand and perform complex bimanual tasks in response to user-specified goals. The project website with code is available at: https://bimanual.github.io} }

Abstract: Bimanual manipulation is challenging due to precise spatial and temporal coordination required between two arms. While there exist several real-world bimanual systems, there is a lack of simulated benchmarks with a large task diversity for systematically studying bimanual capabilities across a wide range of table- top tasks. This paper addresses the gap by extending RLBench to bimanual manipulation. We open-source our code and benchmark, which comprises 13 new tasks with 23 unique task variations, each requiring a high degree of coordination and adaptability. To kickstart the benchmark, we extended several state-of-the-art methods to bimanual manipulation and also present a language-conditioned behavioral cloning agent – PerAct2, an extension of the PerAct framework. This method enables the learning and execution of bimanual 6-DoF manipulation tasks. Our novel network architecture efficiently integrates language processing with action prediction, allowing robots to understand and perform complex bimanual tasks in response to user-specified goals. The project website with code is available at: https://bimanual.github.io

@inproceedings{Amish2024, title = {Johnsen-Rahbek Capstan Clutch: A High Torque Electrostatic Clutch}, author = {Timothy E. Amish and Jeffrey T. Auletta and Chad C. Kessens and Joshua R. Smith and Jeffrey I. Lipton}, year = {2024}, eprint = {2312.12566}, archivePrefix = {arXiv}, primaryClass = {cs.RO}, url = {https://arxiv.org/abs/2312.12566}, file = {https://arxiv.org/pdf/2312.12566}, booktitle = {International Conference on Robotics and Automation (ICRA)}, abstract = {In many robotic systems, the holding state consumes power, limits operating time, and increases operating costs. Electrostatic clutches have the potential to improve robotic performance by generating holding torques with low power consumption. A key limitation of electrostatic clutches has been their low specific shear stresses which restrict generated holding torque, limiting many applications. Here we show how combining the Johnsen-Rahbek (JR) effect with the exponential tension scaling capstan effect can produce clutches with the highest specific shear stress in the literature. Our system generated 31.3 N/cm^2 sheer stress and a total holding torque of 7.1 Nm while consuming only 2.5 mW/cm^2 at 500 V. We demonstrate a theoretical model of an electrostatic adhesive capstan clutch and demonstrate how large angle (theta > 2pi) designs increase efficiency over planar or small angle (theta < pi) clutch designs. We also report the first unfilled polymeric material, polybenzimidazole (PBI), to exhibit the JR-effect.} }

Abstract: In many robotic systems, the holding state consumes power, limits operating time, and increases operating costs. Electrostatic clutches have the potential to improve robotic performance by generating holding torques with low power consumption. A key limitation of electrostatic clutches has been their low specific shear stresses which restrict generated holding torque, limiting many applications. Here we show how combining the Johnsen-Rahbek (JR) effect with the exponential tension scaling capstan effect can produce clutches with the highest specific shear stress in the literature. Our system generated 31.3 N/cm^2 sheer stress and a total holding torque of 7.1 Nm while consuming only 2.5 mW/cm^2 at 500 V. We demonstrate a theoretical model of an electrostatic adhesive capstan clutch and demonstrate how large angle (theta > 2pi) designs increase efficiency over planar or small angle (theta < pi) clutch designs. We also report the first unfilled polymeric material, polybenzimidazole (PBI), to exhibit the JR-effect.

@article{chen2024urdformer, title = {URDFormer: A Pipeline for Constructing Articulated Simulation Environments from Real-World Images}, author = {Zoey Chen and Aaron Walsman and Marius Memmel and Kaichun Mo and Alex Fang and Karthikeya Vemuri and Alan Wu and Dieter Fox and Abhishek Gupta}, journal = {arXiv preprint arXiv:2405.11656}, url = {https://urdformer.github.io/}, file = {https://arxiv.org/pdf/2405.11656}, year = {2024} }

@inproceedings{Li2023, title = {STOW: Discrete-Frame Segmentation and Tracking of Unseen Objects for Warehouse Picking Robots}, year = {2023}, author = {Yi Li and Muru Zhang and Markus Grotz and Kaichun Mo and Dieter Fox}, abstract = {Segmentation and tracking of unseen object instances in discrete frames pose a significant challenge in dynamic industrial robotic contexts, such as distribution warehouses. Here, robots must handle object rearrangements, including shifting, removal, and partial occlusion by new items, and track these items after substantial temporal gaps. The task is further complicated when robots encounter objects beyond their training sets, thereby requiring the ability to segment and track previously unseen items. Considering that continuous observation is often inaccessible in such settings, our task involves working with a discrete set of frames separated by indefinite periods, during which substantial changes to the scene may occur. This task also translates to domestic robotic applications, such as table rearrangement. To address these demanding challenges, we introduce new synthetic and real-world datasets that replicate these industrial and household scenarios. Furthermore, we propose a novel paradigm for joint segmentation and tracking in discrete frames, alongside a transformer module that facilitates efficient inter-frame communication. Our approach significantly outperforms recent methods in our experiments. For additional results and videos, please visit https://sites.google.com/view/stow-corl23.}, url = {https://sites.google.com/view/stow-corl23}, file = {https://arxiv.org/pdf/2311.02337}, booktitle = {Conference on Robot Learning (CoRL)} }

Abstract: Segmentation and tracking of unseen object instances in discrete frames pose a significant challenge in dynamic industrial robotic contexts, such as distribution warehouses. Here, robots must handle object rearrangements, including shifting, removal, and partial occlusion by new items, and track these items after substantial temporal gaps. The task is further complicated when robots encounter objects beyond their training sets, thereby requiring the ability to segment and track previously unseen items. Considering that continuous observation is often inaccessible in such settings, our task involves working with a discrete set of frames separated by indefinite periods, during which substantial changes to the scene may occur. This task also translates to domestic robotic applications, such as table rearrangement. To address these demanding challenges, we introduce new synthetic and real-world datasets that replicate these industrial and household scenarios. Furthermore, we propose a novel paradigm for joint segmentation and tracking in discrete frames, alongside a transformer module that facilitates efficient inter-frame communication. Our approach significantly outperforms recent methods in our experiments. For additional results and videos, please visit https://sites.google.com/view/stow-corl23.

@inproceedings{Yang2023, author = {Boling Yang and Soofiyan Layakalli Atar and Markus Grotz and Byron Boots and Joshua R. Smith}, title = {DYNAMO-GRASP: DYNAMics-aware Optimization for GRASP Point Detection in Suction Grippers}, year = {2023}, abstract = {In this research, we introduce a novel approach to the challenge of suction grasp point detection. Our method, exploiting the strengths of physics-based simulation and data-driven modeling, accounts for object dynamics during the grasping process, markedly enhancing the robot's capability to handle previously unseen objects in real-world scenarios. We benchmark DYNAMO-GRASP against established approaches via comprehensive evaluations in both simulated and real-world environments. Our method outperformed the alternatives, achieving a success rate of 98% in diverse simulated picking tasks and 80% in real-world, adversarially-designed scenarios. Demonstrating a strong ability to adapt to complex and unexpected object dynamics, our method offers robust generalization to real-world challenges. The results of this research set the stage for more reliable and resilient robotic manipulation in intricate real-world situations. Experiment videos and code will be made available at: https://sites.google.com/view/dynamo-grasp}, url = {https://sites.google.com/view/dynamo-grasp}, file = {https://openreview.net/pdf?id=_DYsYC9smK}, booktitle = {Conference on Robot Learning (CoRL)} }

Abstract: In this research, we introduce a novel approach to the challenge of suction grasp point detection. Our method, exploiting the strengths of physics-based simulation and data-driven modeling, accounts for object dynamics during the grasping process, markedly enhancing the robot's capability to handle previously unseen objects in real-world scenarios. We benchmark DYNAMO-GRASP against established approaches via comprehensive evaluations in both simulated and real-world environments. Our method outperformed the alternatives, achieving a success rate of 98% in diverse simulated picking tasks and 80% in real-world, adversarially-designed scenarios. Demonstrating a strong ability to adapt to complex and unexpected object dynamics, our method offers robust generalization to real-world challenges. The results of this research set the stage for more reliable and resilient robotic manipulation in intricate real-world situations. Experiment videos and code will be made available at: https://sites.google.com/view/dynamo-grasp

@inproceedings{Grotz2023rss, title = {Towards robustly picking unseen objects from densely packed shelves}, author = {Markus Grotz, Soofiyan Atar, Yi Li, Paolo Torrado, Boling Yang, Nick Walker, Michael Murray, Maya Cakmak and Joshua R. Smith}, year = {2023}, maintitle = {Robotics Science and Systems (RSS)}, booktitle = {Workshop on Perception and Manipulation Challenges for Warehouse Automation}, abstract = {For industrial warehouses to be fully automated, robots must be able to pick previously unseen objects from densely packed shelves. Dense packing makes objects more difficult to distinguish visually and requires manipulation to be sensitive to the arrangement of objects in the shelf itself. Hence, key challenges that arise are the visual segmentation and tracking of previously unseen objects in cluttered environments, as well as manipulation planning and control to pick densely packed objects from the shelf. We present a complete system that is able to pick unseen objects from a cluttered shelf. Our system consists of components to track shelf inventory, re-identify requested objects and to autonomously pick them. In our experiments, the system is able to pick objects in highly cluttered scenes with a success rate of 66% and 53.8 successful picks per hour. We provide a classification of the pick attempts and their frequency to motivate future research.}, file = {pdf:2023-grotz_rss.pdf}, url = {http://armbench.s3.amazonaws.com/rss23.html} }

Abstract: For industrial warehouses to be fully automated, robots must be able to pick previously unseen objects from densely packed shelves. Dense packing makes objects more difficult to distinguish visually and requires manipulation to be sensitive to the arrangement of objects in the shelf itself. Hence, key challenges that arise are the visual segmentation and tracking of previously unseen objects in cluttered environments, as well as manipulation planning and control to pick densely packed objects from the shelf. We present a complete system that is able to pick unseen objects from a cluttered shelf. Our system consists of components to track shelf inventory, re-identify requested objects and to autonomously pick them. In our experiments, the system is able to pick objects in highly cluttered scenes with a success rate of 66% and 53.8 successful picks per hour. We provide a classification of the pick attempts and their frequency to motivate future research.