-
Fighting Failures with FIRE: Failure Identification to Reduce Expert Burden in Intervention-Based Learning
T. Ablett, F. Maric, and J. Kelly
Toronto, Ontario, Canada, Tech. Rep. STARS-20-001, Aug. 10, 2020. Bibtex |
Abstract |
arXiv @techreport{2020_Ablett_Fighting,
abstract = {Supervised imitation learning, also known as behavioral cloning, suffers from distribution drift leading to failures during policy execution. One approach to mitigate this issue is to allow an expert to correct the agent's actions during task execution, based on the expert's determina- tion that the agent has reached a `point of no return.' The agent's policy is then retrained using this new corrective data. This approach alone can enable high-performance agents to be learned, but at a substantial cost: the expert must vigilantly observe execution until the policy reaches a specified level of success, and even at that point, there is no guarantee that the policy will always succeed. To address these limitations, we present FIRE (Failure Identification to Reduce Expert burden), a system that can predict when a running policy will fail, halt its execution, and request a correction from the expert. Unlike existing approaches that learn only from expert data, our approach learns from both expert and non-expert data, akin to adversarial learning. We demonstrate experimentally for a series of challenging manipulation tasks that our method is able to recognize state-action pairs that lead to failures. This permits seamless integration into an intervention-based learning system, where we show an order-of-magnitude gain in sample efficiency compared with a state-of-the-art inverse reinforcement learning method and dramatically improved performance over an equivalent amount of data learned with behavioral cloning.},
address = {Toronto, Ontario, Canada},
author = {Trevor Ablett and Filip Maric and Jonathan Kelly},
date = {2020-08-10},
institution = {University of Toronto},
month = {Aug. 10},
number = {STARS-20-001},
title = {Fighting Failures with {FIRE}: Failure Identification to Reduce Expert Burden in Intervention-Based Learning},
url = {https://arxiv.org/abs/2007.00245},
year = {2020}
}
Supervised imitation learning, also known as behavioral cloning, suffers from distribution drift leading to failures during policy execution. One approach to mitigate this issue is to allow an expert to correct the agent's actions during task execution, based on the expert's determina- tion that the agent has reached a `point of no return.' The agent's policy is then retrained using this new corrective data. This approach alone can enable high-performance agents to be learned, but at a substantial cost: the expert must vigilantly observe execution until the policy reaches a specified level of success, and even at that point, there is no guarantee that the policy will always succeed. To address these limitations, we present FIRE (Failure Identification to Reduce Expert burden), a system that can predict when a running policy will fail, halt its execution, and request a correction from the expert. Unlike existing approaches that learn only from expert data, our approach learns from both expert and non-expert data, akin to adversarial learning. We demonstrate experimentally for a series of challenging manipulation tasks that our method is able to recognize state-action pairs that lead to failures. This permits seamless integration into an intervention-based learning system, where we show an order-of-magnitude gain in sample efficiency compared with a state-of-the-art inverse reinforcement learning method and dramatically improved performance over an equivalent amount of data learned with behavioral cloning.
-
Learning Matchable Image Transformations for Long-term Visual Localization
L. Clement, M. Gridseth, J. Tomasi, and J. Kelly
IEEE Robotics and Automation Letters, vol. 5, iss. 2, pp. 1492-1499, 2020. DOI |
Bibtex |
Abstract |
arXiv |
Video @article{2020_Clement_Learning_A,
abstract = {Long-term metric self-localization is an essential capability of autonomous mobile robots, but remains challenging for vision-based systems due to appearance changes caused by lighting, weather, or seasonal variations. While experience-based mapping has proven to be an effective technique for bridging the `appearance gap,' the number of experiences required for reliable metric localization over days or months can be very large, and methods for reducing the necessary number of experiences are needed for this approach to scale. Taking inspiration from color constancy theory, we learn a nonlinear RGB-to-grayscale mapping that explicitly maximizes the number of inlier feature matches for images captured under different lighting and weather conditions, and use it as a pre-processing step in a conventional single-experience localization pipeline to improve its robustness to appearance change. We train this mapping by approximating the target non-differentiable localization pipeline with a deep neural network, and find that incorporating a learned low-dimensional context feature can further improve cross-appearance feature matching. Using synthetic and real-world datasets, we demonstrate substantial improvements in localization performance across day-night cycles, enabling continuous metric localization over a 30-hour period using a single mapping experience, and allowing experience-based localization to scale to long deployments with dramatically reduced data requirements.},
author = {Lee Clement and Mona Gridseth and Justin Tomasi and Jonathan Kelly},
doi = {10.1109/LRA.2020.2967659},
journal = {{IEEE} Robotics and Automation Letters},
month = {April},
number = {2},
pages = {1492--1499},
title = {Learning Matchable Image Transformations for Long-term Visual Localization},
url = {https://arxiv.org/abs/1904.01080},
video1 = {https://www.youtube.com/watch?v=WrxaSpHKxE8&t=1s},
volume = {5},
year = {2020}
}
Long-term metric self-localization is an essential capability of autonomous mobile robots, but remains challenging for vision-based systems due to appearance changes caused by lighting, weather, or seasonal variations. While experience-based mapping has proven to be an effective technique for bridging the `appearance gap,' the number of experiences required for reliable metric localization over days or months can be very large, and methods for reducing the necessary number of experiences are needed for this approach to scale. Taking inspiration from color constancy theory, we learn a nonlinear RGB-to-grayscale mapping that explicitly maximizes the number of inlier feature matches for images captured under different lighting and weather conditions, and use it as a pre-processing step in a conventional single-experience localization pipeline to improve its robustness to appearance change. We train this mapping by approximating the target non-differentiable localization pipeline with a deep neural network, and find that incorporating a learned low-dimensional context feature can further improve cross-appearance feature matching. Using synthetic and real-world datasets, we demonstrate substantial improvements in localization performance across day-night cycles, enabling continuous metric localization over a 30-hour period using a single mapping experience, and allowing experience-based localization to scale to long deployments with dramatically reduced data requirements.
-
On Learning Models of Appearance for Robust Long-term Visual Navigation
L. E. Clement
PhD Thesis , University of Toronto, Toronto, Ontario, Canada, 2020. Bibtex |
Abstract |
PDF @phdthesis{2020_Clement_Learning_B,
abstract = {Simultaneous localization and mapping (SLAM) is a class of techniques that allow robots to navigate unknown environments using onboard sensors. With inexpensive commercial cameras as the primary sensor, visual SLAM has become an important and widely used approach to enabling mobile robot autonomy. However, traditional visual SLAM algorithms make use of only a fraction of the information available from conventional cameras: in addition to the basic geometric cues typically used in visual SLAM, colour images encode a wealth of information about the camera, environmental illumination, surface materials, vehicle motion, and other factors influencing the image formation process. Moreover, visual localization performance degrades quickly in long-term deployments due to environmental appearance changes caused by lighting, weather, or seasonal effects. This is especially problematic when continuous metric localization is required to drive vision-in-the-loop systems such as autonomous route following. This thesis explores several novel approaches to exploiting additional information from vision sensors in order to improve the accuracy and reliability of metric visual SLAM algorithms in short- and long-term deployments. First, we develop a technique for reducing drift error in visual odometry (VO) by estimating the position of a known light source such as the sun using indirect illumination cues available from existing image streams. We build and evaluate hand-engineered and learned models for single-image sun detection and achieve significant reductions in drift error over 30 km of driving in urban and planetary analogue environments. Second, we explore deep image-to-image translation as a means of improving metric visual localization under time-varying illumination. Using images captured under different illumination conditions in a common environment, we demonstrate that localization accuracy and reliability can be substantially improved by learning a many-to-one mapping to a user-selected canonical appearance condition. Finally, we develop a self-supervised method for learning a canonical appearance optimized for high-quality localization. By defining a differentiable surrogate loss function related to the performance of a non-differentiable localization pipeline, we train an optimal RGB-to-grayscale mapping for a given environment, sensor, and pipeline. Using synthetic and real-world long-term vision datasets, we demonstrate significant improvements in localization performance compared to standard grayscale images, enabling continuous metric localization over day-night cycles using a single mapping experience.},
address = {Toronto, Ontario, Canada},
author = {Lee Eric Clement},
institution = {University of Toronto},
month = {January},
school = {University of Toronto},
title = {On Learning Models of Appearance for Robust Long-term Visual Navigation},
year = {2020}
}
Simultaneous localization and mapping (SLAM) is a class of techniques that allow robots to navigate unknown environments using onboard sensors. With inexpensive commercial cameras as the primary sensor, visual SLAM has become an important and widely used approach to enabling mobile robot autonomy. However, traditional visual SLAM algorithms make use of only a fraction of the information available from conventional cameras: in addition to the basic geometric cues typically used in visual SLAM, colour images encode a wealth of information about the camera, environmental illumination, surface materials, vehicle motion, and other factors influencing the image formation process. Moreover, visual localization performance degrades quickly in long-term deployments due to environmental appearance changes caused by lighting, weather, or seasonal effects. This is especially problematic when continuous metric localization is required to drive vision-in-the-loop systems such as autonomous route following. This thesis explores several novel approaches to exploiting additional information from vision sensors in order to improve the accuracy and reliability of metric visual SLAM algorithms in short- and long-term deployments. First, we develop a technique for reducing drift error in visual odometry (VO) by estimating the position of a known light source such as the sun using indirect illumination cues available from existing image streams. We build and evaluate hand-engineered and learned models for single-image sun detection and achieve significant reductions in drift error over 30 km of driving in urban and planetary analogue environments. Second, we explore deep image-to-image translation as a means of improving metric visual localization under time-varying illumination. Using images captured under different illumination conditions in a common environment, we demonstrate that localization accuracy and reliability can be substantially improved by learning a many-to-one mapping to a user-selected canonical appearance condition. Finally, we develop a self-supervised method for learning a canonical appearance optimized for high-quality localization. By defining a differentiable surrogate loss function related to the performance of a non-differentiable localization pipeline, we train an optimal RGB-to-grayscale mapping for a given environment, sensor, and pipeline. Using synthetic and real-world long-term vision datasets, we demonstrate significant improvements in localization performance compared to standard grayscale images, enabling continuous metric localization over day-night cycles using a single mapping experience.
-
The Canadian Planetary Emulation Terrain Energy-Aware Rover Navigation Dataset
O. Lamarre, O. Limoyo, F. Maric, and J. Kelly
The International Journal of Robotics Research, vol. 39, iss. 6, pp. 641-650, 2020. DOI |
Bibtex |
Abstract |
Code @article{2020_Lamarre_Canadian,
abstract = {Future exploratory missions to the Moon and to Mars will involve solar-powered rovers; careful vehicle energy management is critical to the success of such missions. This article describes a unique dataset gathered by a small, four-wheeled rover at a planetary analog test facility in Canada. The rover was equipped with a suite of sensors designed to enable the study of energy-aware navigation and path planning algorithms. The sensors included a colour omnidirectional stereo camera, a monocular camera, an inertial measurement unit, a pyranometer, drive power consumption monitors, wheel encoders, and a GPS receiver. In total, the rover drove more than 1.2 km over varied terrain at the analog test site. All data is presented in human-readable text files and as standard-format images; additional Robot Operating System (ROS) parsing tools and several georeferenced aerial maps of the test environment are also included. A series of potential research use cases is described.},
author = {Olivier Lamarre and Oliver Limoyo and Filip Maric and Jonathan Kelly},
code = {https://github.com/utiasSTARS/enav-planetary-dataset},
doi = {10.1177/0278364920908922},
journal = {The International Journal of Robotics Research},
month = {May},
number = {6},
pages = {641--650},
title = {The Canadian Planetary Emulation Terrain Energy-Aware Rover Navigation Dataset},
volume = {39},
year = {2020}
}
Future exploratory missions to the Moon and to Mars will involve solar-powered rovers; careful vehicle energy management is critical to the success of such missions. This article describes a unique dataset gathered by a small, four-wheeled rover at a planetary analog test facility in Canada. The rover was equipped with a suite of sensors designed to enable the study of energy-aware navigation and path planning algorithms. The sensors included a colour omnidirectional stereo camera, a monocular camera, an inertial measurement unit, a pyranometer, drive power consumption monitors, wheel encoders, and a GPS receiver. In total, the rover drove more than 1.2 km over varied terrain at the analog test site. All data is presented in human-readable text files and as standard-format images; additional Robot Operating System (ROS) parsing tools and several georeferenced aerial maps of the test environment are also included. A series of potential research use cases is described.
-
Impact of Traversability Uncertainty on Global Navigation Planning in Planetary Environments
O. Lamarre, A. B. Asghar, and J. Kelly
Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS’20) Workshop on Planetary Exploration Robots, Las Vegas, Nevada, USA, Oct. 29, 2020. DOI |
Bibtex @inproceedings{2020_Lamarre_Impact,
address = {Las Vegas, Nevada, USA},
author = {Olivier Lamarre and Ahmad Bilal Asghar and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE/RSJ} International Conference on Intelligent Robots and Systems {(IROS'20)} Workshop on Planetary Exploration Robots},
date = {2020-10-29},
doi = {10.3929/ethz-b-000450119},
month = {Oct. 29},
note = {Moog Workshop Poster Competition First Prize},
title = {Impact of Traversability Uncertainty on Global Navigation Planning in Planetary Environments},
year = {2020}
}
Moog Workshop Poster Competition First Prize
-
Heteroscedastic Uncertainty for Robust Generative Latent Dynamics
O. Limoyo, B. Chan, F. Maric, B. Wagstaff, R. Mahmood, and J. Kelly
IEEE Robotics and Automation Letters, vol. 5, iss. 4, pp. 6654-6661, 2020. DOI |
Bibtex |
Abstract |
arXiv |
Video @article{2020_Limoyo_Heteroscedastic,
abstract = {Learning or identifying dynamics from a sequence of high-dimensional observations is a difficult challenge in many domains, including reinforcement learning and control. The problem has recently been studied from a generative perspective through latent dynamics: high-dimensional observations are embedded into a lower-dimensional space in which the dynamics can be learned. Despite some successes, latent dynamics models have not yet been applied to real-world robotic systems where learned representations must be robust to a variety of perceptual confounds and noise sources not seen during training. In this paper, we present a method to jointly learn a latent state representation and the associated dynamics that is amenable for long-term planning and closed-loop control under perceptually difficult conditions. As our main contribution, we describe how our representation is able to capture a notion of heteroscedastic or input-specific uncertainty at test time by detecting novel or out-of-distribution (OOD) inputs. We present results from prediction and control experiments on two image-based tasks: a simulated pendulum balancing task and a real-world robotic manipulator reaching task. We demonstrate that our model produces significantly more accurate predictions and exhibits improved control performance, compared to a model that assumes homoscedastic uncertainty only, in the presence of varying degrees of input degradation.},
author = {Oliver Limoyo and Bryan Chan and Filip Maric and Brandon Wagstaff and Rupam Mahmood and Jonathan Kelly},
doi = {10.1109/LRA.2020.3015449},
journal = {{IEEE} Robotics and Automation Letters},
month = {October},
number = {4},
pages = {6654--6661},
title = {Heteroscedastic Uncertainty for Robust Generative Latent Dynamics},
url = {https://arxiv.org/abs/2008.08157},
video1 = {https://www.youtube.com/watch?v=tPLUqhobVzw},
volume = {5},
year = {2020}
}
Learning or identifying dynamics from a sequence of high-dimensional observations is a difficult challenge in many domains, including reinforcement learning and control. The problem has recently been studied from a generative perspective through latent dynamics: high-dimensional observations are embedded into a lower-dimensional space in which the dynamics can be learned. Despite some successes, latent dynamics models have not yet been applied to real-world robotic systems where learned representations must be robust to a variety of perceptual confounds and noise sources not seen during training. In this paper, we present a method to jointly learn a latent state representation and the associated dynamics that is amenable for long-term planning and closed-loop control under perceptually difficult conditions. As our main contribution, we describe how our representation is able to capture a notion of heteroscedastic or input-specific uncertainty at test time by detecting novel or out-of-distribution (OOD) inputs. We present results from prediction and control experiments on two image-based tasks: a simulated pendulum balancing task and a real-world robotic manipulator reaching task. We demonstrate that our model produces significantly more accurate predictions and exhibits improved control performance, compared to a model that assumes homoscedastic uncertainty only, in the presence of varying degrees of input degradation.
-
Geometry-Aware Singularity Avoidance for Articulated Robots Using a Riemannian Metric
F. Maric, L. Petrovic, M. Guberina, J. Kelly, and I. Petrovic
Robotics and Autonomous Systems, 2020. Bibtex @article{2020_Maric_Geometry-Aware,
author = {Filip Maric and Luka Petrovic and Marko Guberina and Jonathan Kelly and Ivan Petrovic},
journal = {Robotics and Autonomous Systems},
note = {Submitted},
title = {Geometry-Aware Singularity Avoidance for Articulated Robots Using a Riemannian Metric},
year = {2020}
}
Submitted
-
Inverse Kinematics for Serial Kinematic Chains via Sum of Squares Optimization
F. Maric, M. Giamou, S. Khoubyarian, I. Petrovic, and J. Kelly
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’20), Paris, France, May 31–Jun. 4, 2020, pp. 7101-7107. DOI |
Bibtex |
Abstract |
arXiv |
Video |
Code @inproceedings{2020_Maric_Inverse_A,
abstract = {Inverse kinematics is a fundamental challenge for articulated robots: fast and accurate algorithms are needed for translating task-related workspace constraints and goals into feasible joint configurations. In general, inverse kinematics for serial kinematic chains is a difficult nonlinear problem, for which closed form solutions cannot easily be obtained. Therefore, computationally efficient numerical methods that can be adapted to a general class of manipulators are of great importance. In this paper, we use convex optimization techniques to solve the inverse kinematics problem with joint limit constraints for highly redundant serial kinematic chains with spherical joints in two and three dimensions. This is accomplished through a novel formulation of inverse kinematics as a nearest point problem, and with a fast sum of squares solver that exploits the sparsity of kinematic constraints for serial manipulators. Our method has the advantages of post-hoc certification of global optimality and a runtime that scales polynomially with the number of degrees of freedom. Additionally, we prove that our convex relaxation leads to a globally optimal solution when certain conditions are met, and demonstrate empirically that these conditions are common and represent many practical instances. Finally, we provide an open source implementation of our algorithm.},
address = {Paris, France},
author = {Filip Maric and Matthew Giamou and Soroush Khoubyarian and Ivan Petrovic and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'20})},
code = {https://github.com/utiasSTARS/sos-ik},
date = {2020-05-31/2020-06-04},
doi = {10.1109/ICRA40945.2020.9196704},
month = {May 31--Jun. 4},
pages = {7101--7107},
title = {Inverse Kinematics for Serial Kinematic Chains via Sum of Squares Optimization},
url = {http://arxiv.org/abs/1909.09318},
video1 = {https://www.youtube.com/watch?v=AdPze8cTUuE},
year = {2020}
}
Inverse kinematics is a fundamental challenge for articulated robots: fast and accurate algorithms are needed for translating task-related workspace constraints and goals into feasible joint configurations. In general, inverse kinematics for serial kinematic chains is a difficult nonlinear problem, for which closed form solutions cannot easily be obtained. Therefore, computationally efficient numerical methods that can be adapted to a general class of manipulators are of great importance. In this paper, we use convex optimization techniques to solve the inverse kinematics problem with joint limit constraints for highly redundant serial kinematic chains with spherical joints in two and three dimensions. This is accomplished through a novel formulation of inverse kinematics as a nearest point problem, and with a fast sum of squares solver that exploits the sparsity of kinematic constraints for serial manipulators. Our method has the advantages of post-hoc certification of global optimality and a runtime that scales polynomially with the number of degrees of freedom. Additionally, we prove that our convex relaxation leads to a globally optimal solution when certain conditions are met, and demonstrate empirically that these conditions are common and represent many practical instances. Finally, we provide an open source implementation of our algorithm.
-
Inverse Kinematics as Low-Rank Euclidean Distance Matrix Completion
F. Maric, M. Giamou, I. Petrovic, and J. Kelly
Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS’20) Workshop on Bringing Geometric Methods to Robot Learning, Optimization and Control, Las Vegas, Nevada, USA, Oct. 29, 2020. Bibtex |
Abstract |
arXiv |
Video @inproceedings{2020_Maric_Inverse_B,
abstract = {The majority of inverse kinematics (IK) algorithms search for solutions in a configuration space defined by joint angles. However, the kinematics of many robots can also be described in terms of distances between rigidly-attached points, which collectively form a Euclidean distance matrix. This alternative geometric description of the kinematics reveals an elegant equivalence between IK and the problem of low-rank matrix completion. We use this connection to implement a novel Riemannian optimization-based solution to IK for various articulated robots with symmetric joint angle constraints.},
address = {Las Vegas, Nevada, USA},
author = {Filip Maric and Matthew Giamou and Ivan Petrovic and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE/RSJ} International Conference on Intelligent Robots and Systems {(IROS'20)} Workshop on Bringing Geometric Methods to Robot Learning, Optimization and Control},
date = {2020-10-29},
month = {Oct. 29},
note = {Bosch Center for Artificial Intelligence Best Workshop Contribution Award},
title = {Inverse Kinematics as Low-Rank Euclidean Distance Matrix Completion},
url = {https://arxiv.org/abs/2011.04850},
video1 = {https://www.youtube.com/watch?v=wO0_w2Gw5jk&t=10s},
year = {2020}
}
The majority of inverse kinematics (IK) algorithms search for solutions in a configuration space defined by joint angles. However, the kinematics of many robots can also be described in terms of distances between rigidly-attached points, which collectively form a Euclidean distance matrix. This alternative geometric description of the kinematics reveals an elegant equivalence between IK and the problem of low-rank matrix completion. We use this connection to implement a novel Riemannian optimization-based solution to IK for various articulated robots with symmetric joint angle constraints.
Bosch Center for Artificial Intelligence Best Workshop Contribution Award
-
Riemannian Optimization for Distance Geometric Inverse Kinematics
F. Maric, M. Giamou, A. Hall, S. Khoubyarian, I. Petrovic, and J. Kelly
IEEE Transactions on Robotics, 2020. Bibtex |
Abstract @article{2020_Maric_Riemannian,
abstract = {Solving the inverse kinematics problem is a fundamental challenge in motion planning, control, and calibration for articulated robots. Kinematic models for these robots are typically param- eterized by joint angles, generating a complicated trigonometric mapping between a robot's configuration and end-effector pose. Alternatively, the kinematic model and task constraints can be represented using invariant distances between points attached to the robot. In this paper, we prove the equivalence of distance-based inverse kinematics formulations and the distance geometry problem for a large class of robots comprised of revolute joints. Unlike previous approaches, we use the connection between distance geometry and low-rank matrix completion to find inverse kinematics solutions by completing a partial Euclidean distance matrix using local optimization. Further, we parameterize the space of Euclidean distance matrices with the Riemannian manifold of fixed-rank Gram matrices, allowing us to leverage a variety of mature Riemannian optimization methods. Finally, we show that bound smoothing can be used to generate informed initializations without significant computational overhead, improving convergence. We demonstrate that our novel inverse kinematics solver achieves higher success rates compared to traditional approaches, and significantly outperforms them in many cases where multiple end-effectors are present.},
author = {Filip Maric and Matthew Giamou and Adam Hall and Soroush Khoubyarian and Ivan Petrovic and Jonathan Kelly},
journal = {{IEEE} Transactions on Robotics},
note = {Submitted},
title = {Riemannian Optimization for Distance Geometric Inverse Kinematics},
year = {2020}
}
Solving the inverse kinematics problem is a fundamental challenge in motion planning, control, and calibration for articulated robots. Kinematic models for these robots are typically param- eterized by joint angles, generating a complicated trigonometric mapping between a robot's configuration and end-effector pose. Alternatively, the kinematic model and task constraints can be represented using invariant distances between points attached to the robot. In this paper, we prove the equivalence of distance-based inverse kinematics formulations and the distance geometry problem for a large class of robots comprised of revolute joints. Unlike previous approaches, we use the connection between distance geometry and low-rank matrix completion to find inverse kinematics solutions by completing a partial Euclidean distance matrix using local optimization. Further, we parameterize the space of Euclidean distance matrices with the Riemannian manifold of fixed-rank Gram matrices, allowing us to leverage a variety of mature Riemannian optimization methods. Finally, we show that bound smoothing can be used to generate informed initializations without significant computational overhead, improving convergence. We demonstrate that our novel inverse kinematics solver achieves higher success rates compared to traditional approaches, and significantly outperforms them in many cases where multiple end-effectors are present.
Submitted
-
Unified Spatiotemporal Calibration of Monocular Cameras and Planar Lidars
J. Marr and J. Kelly
in Proceedings of the 2018 International Symposium on Experimental Robotics , J. Xiao, T. Kroger, and O. Khatib, Eds., Cham: Springer International Publishing AG, 2020, vol. 11, pp. 781-790. DOI |
Bibtex |
Abstract @incollection{2020_Marr_Unified,
abstract = {Monocular cameras and planar lidar sensors are complementary. While monocular visual odometry (VO) is a relatively low-drift method for measuring platform egomotion, it suffers from a scale ambiguity. A planar lidar scanner, in contrast, is able to provide precise distance information with known scale. In combination, a monocular camera-2D lidar pair can be used as a performance 3D scanner, at a much lower cost than existing 3D lidar units. However, for accurate scan acquisition, the two sensors must be spatially and temporally calibrated. In this paper, we extend recent work on a calibration technique based on R ́enyi's quadratic entropy (RQE) to the unified spatiotemporal calibration of monocular cameras and 2D lidars. We present simulation results indicating that calibration errors of less than 5 mm, 0.1 degrees, and 0.15 ms in translation, rotation, and time delay, respectively, are readily achievable. Using real-world data, in the absence of reliable ground truth, we demonstrate high repeatability given sufficient platform motion. Unlike existing techniques, we are able to calibrate in arbitrary, target-free environments and without the need for overlapping sensor fields of view.},
address = {Cham},
author = {Jordan Marr and Jonathan Kelly},
booktitle = {Proceedings of the 2018 International Symposium on Experimental Robotics},
doi = {10.1007/978-3-030-33950-0_67},
editor = {Jing Xiao and Torsten Kroger and Oussama Khatib},
isbn = {978-3-030-33949-4},
pages = {781--790},
publisher = {Springer International Publishing AG},
series = {Springer Proceedings in Advanced Robotics},
title = {Unified Spatiotemporal Calibration of Monocular Cameras and Planar Lidars},
volume = {11},
year = {2020}
}
Monocular cameras and planar lidar sensors are complementary. While monocular visual odometry (VO) is a relatively low-drift method for measuring platform egomotion, it suffers from a scale ambiguity. A planar lidar scanner, in contrast, is able to provide precise distance information with known scale. In combination, a monocular camera-2D lidar pair can be used as a performance 3D scanner, at a much lower cost than existing 3D lidar units. However, for accurate scan acquisition, the two sensors must be spatially and temporally calibrated. In this paper, we extend recent work on a calibration technique based on R ́enyi's quadratic entropy (RQE) to the unified spatiotemporal calibration of monocular cameras and 2D lidars. We present simulation results indicating that calibration errors of less than 5 mm, 0.1 degrees, and 0.15 ms in translation, rotation, and time delay, respectively, are readily achievable. Using real-world data, in the absence of reliable ground truth, we demonstrate high repeatability given sufficient platform motion. Unlike existing techniques, we are able to calibrate in arbitrary, target-free environments and without the need for overlapping sensor fields of view.
-
Towards a Policy-as-a-Service Framework to Enable Compliant, Trustworthy AI and HRI Systems in the Wild
A. Morris, H. Siegel, and J. Kelly
Proceedings of the AAAI Fall Symposium on Artificial Intelligence for Human-Robot Interaction: Trust & Explainability in Artificial Intelligence for Human-Robot Interaction (AI-HRI’20), Arlington, Virginia, USA, Nov. 13–14, 2020. Bibtex |
Abstract |
arXiv @inproceedings{2020_Morris_Towards,
abstract = {Building trustworthy autonomous systems is challenging for many reasons beyond simply trying to engineer agents that 'always do the right thing.' There is a broader context that is often not considered within AI and HRI: that the problem of trustworthiness is inherently socio-technical and ultimately involves a broad set of complex human factors and multidimensional relationships that can arise between agents, humans, organizations, and even governments and legal institutions, each with their own understanding and definitions of trust. This complexity presents a significant barrier to the development of trustworthy AI and HRI systems---while systems developers may desire to have their systems 'always do the right thing,' they generally lack the practical tools and expertise in law, regulation, policy and ethics to ensure this outcome. In this paper, we emphasize the "fuzzy" socio-technical aspects of trustworthiness and the need for their careful consideration during both design and deployment. We hope to contribute to the discussion of trustworthy engineering in AI and HRI by i) describing the policy landscape that must be considered when addressing trustworthy computing and the need for usable trust models, ii) highlighting an opportunity for trustworthy-by-design intervention within the systems engineering process, and iii) introducing the concept of a "policy-as-a-service" (PaaS) framework that can be readily applied by AI systems engineers to address the fuzzy problem of trust during the development and (eventually) runtime process. We envision that the PaaS approach, which offloads the development of policy design parameters and maintenance of policy standards to policy experts, will enable runtime trust capabilities intelligent systems in the wild.},
address = {Arlington, Virginia, USA},
author = {Alexis Morris and Hallie Siegel and Jonathan Kelly},
booktitle = {Proceedings of the AAAI Fall Symposium on Artificial Intelligence for Human-Robot Interaction: Trust \& Explainability in Artificial Intelligence for Human-Robot Interaction {(AI-HRI'20)}},
date = {2020-11-13/2020-11-14},
month = {Nov. 13--14},
title = {Towards a Policy-as-a-Service Framework to Enable Compliant, Trustworthy AI and HRI Systems in the Wild},
url = {https://arxiv.org/abs/2010.07022},
year = {2020}
}
Building trustworthy autonomous systems is challenging for many reasons beyond simply trying to engineer agents that 'always do the right thing.' There is a broader context that is often not considered within AI and HRI: that the problem of trustworthiness is inherently socio-technical and ultimately involves a broad set of complex human factors and multidimensional relationships that can arise between agents, humans, organizations, and even governments and legal institutions, each with their own understanding and definitions of trust. This complexity presents a significant barrier to the development of trustworthy AI and HRI systems---while systems developers may desire to have their systems 'always do the right thing,' they generally lack the practical tools and expertise in law, regulation, policy and ethics to ensure this outcome. In this paper, we emphasize the "fuzzy" socio-technical aspects of trustworthiness and the need for their careful consideration during both design and deployment. We hope to contribute to the discussion of trustworthy engineering in AI and HRI by i) describing the policy landscape that must be considered when addressing trustworthy computing and the need for usable trust models, ii) highlighting an opportunity for trustworthy-by-design intervention within the systems engineering process, and iii) introducing the concept of a "policy-as-a-service" (PaaS) framework that can be readily applied by AI systems engineers to address the fuzzy problem of trust during the development and (eventually) runtime process. We envision that the PaaS approach, which offloads the development of policy design parameters and maintenance of policy standards to policy experts, will enable runtime trust capabilities intelligent systems in the wild.
-
Learned Improvements to the Visual Egomotion Pipeline
V. Peretroukhin
PhD Thesis , University of Toronto, Toronto, Ontario, Canada, 2020. Bibtex |
Abstract |
PDF @phdthesis{2020_Peretroukhin_Learned,
abstract = {The ability to estimate egomotion is at the heart of safe and reliable mobile autonomy. By inferring pose changes from sequential sensor measurements, egomotion estimation forms the basis of mapping and navigation pipelines, and permits mobile robots to self-localize within environments where external localization information may be intermittent or unavailable. Visual egomotion estimation, also known as visual odometry, has become ubiquitous in mobile robotics due to the availability of high-quality, compact, and inexpensive cameras that capture rich representations of the world. Classical visual odometry pipelines make simplifying assumptions that, while permitting reliable operation in ideal conditions, often lead to systematic error. In this dissertation, we present four ways in which conventional pipelines can be improved through the addition of a learned hyper-parametric model. By combining traditional pipelines with learning, we retain the performance of conventional techniques in nominal conditions while leveraging modern high-capacity data-driven models to improve uncertainty quantification, correct for systematic bias, and improve robustness to deleterious effects by extracting latent information in existing visual data. We demonstrate the improvements derived from our approach on data collected in sundry settings such as urban roads, indoor labs, and planetary analogue sites in the Canadian High Arctic.},
address = {Toronto, Ontario, Canada},
author = {Valentin Peretroukhin},
institution = {University of Toronto},
month = {March},
school = {University of Toronto},
title = {Learned Improvements to the Visual Egomotion Pipeline},
year = {2020}
}
The ability to estimate egomotion is at the heart of safe and reliable mobile autonomy. By inferring pose changes from sequential sensor measurements, egomotion estimation forms the basis of mapping and navigation pipelines, and permits mobile robots to self-localize within environments where external localization information may be intermittent or unavailable. Visual egomotion estimation, also known as visual odometry, has become ubiquitous in mobile robotics due to the availability of high-quality, compact, and inexpensive cameras that capture rich representations of the world. Classical visual odometry pipelines make simplifying assumptions that, while permitting reliable operation in ideal conditions, often lead to systematic error. In this dissertation, we present four ways in which conventional pipelines can be improved through the addition of a learned hyper-parametric model. By combining traditional pipelines with learning, we retain the performance of conventional techniques in nominal conditions while leveraging modern high-capacity data-driven models to improve uncertainty quantification, correct for systematic bias, and improve robustness to deleterious effects by extracting latent information in existing visual data. We demonstrate the improvements derived from our approach on data collected in sundry settings such as urban roads, indoor labs, and planetary analogue sites in the Canadian High Arctic.
-
A Smooth Representation of SO(3) for Deep Rotation Learning with Uncertainty
V. Peretroukhin, M. Giamou, D. Rosen, N. W. Greene, N. Roy, and J. Kelly
Proceedings of Robotics: Science and Systems (RSS’20), Corvallis, Oregon, USA, Jul. 12–16, 2020. DOI |
Bibtex |
Abstract |
arXiv |
Site |
Code @inproceedings{2020_Peretroukhin_Smooth,
abstract = {Accurate rotation estimation is at the heart of robot perception tasks such as visual odometry and object pose estimation. Deep neural networks have provided a new way to perform these tasks, and the choice of rotation representation is an important part of network design. In this work, we present a novel symmetric matrix representation of the 3D rotation group, SO(3), with two important properties that make it particularly suitable for learned models: (1) it satisfies a smoothness property that improves convergence and generalization when regressing large rotation targets, and (2) it encodes a symmetric Bingham belief over the space of unit quaternions, permitting the training of uncertainty-aware models. We empirically validate the benefits of our formulation by training deep neural rotation regressors on two data modalities. First, we use synthetic point-cloud data to show that our representation leads to superior predictive accuracy over existing representations for arbitrary rotation targets. Second, we use image data collected onboard ground and aerial vehicles to demonstrate that our representation is amenable to an effective out-of-distribution (OOD) rejection technique that significantly improves the robustness of rotation estimates to unseen environmental effects and corrupted input images, without requiring the use of an explicit likelihood loss, stochastic sampling, or an auxiliary classifier. This capability is key for safety-critical applications where detecting novel inputs can prevent catastrophic failure of learned models.},
address = {Corvallis, Oregon, USA},
author = {Valentin Peretroukhin and Matthew Giamou and David Rosen and W. Nicholas Greene and Nicholas Roy and Jonathan Kelly},
booktitle = {Proceedings of Robotics: Science and Systems {(RSS'20)}},
code = {https://github.com/utiasSTARS/bingham-rotation-learning},
date = {2020-07-12/2020-07-16},
doi = {10.15607/RSS.2020.XVI.007},
month = {Jul. 12--16},
note = {Best Student Paper Award},
site = {https://papers.starslab.ca/bingham-rotation-learning/},
title = {A Smooth Representation of SO(3) for Deep Rotation Learning with Uncertainty},
url = {https://arxiv.org/abs/2006.01031},
year = {2020}
}
Accurate rotation estimation is at the heart of robot perception tasks such as visual odometry and object pose estimation. Deep neural networks have provided a new way to perform these tasks, and the choice of rotation representation is an important part of network design. In this work, we present a novel symmetric matrix representation of the 3D rotation group, SO(3), with two important properties that make it particularly suitable for learned models: (1) it satisfies a smoothness property that improves convergence and generalization when regressing large rotation targets, and (2) it encodes a symmetric Bingham belief over the space of unit quaternions, permitting the training of uncertainty-aware models. We empirically validate the benefits of our formulation by training deep neural rotation regressors on two data modalities. First, we use synthetic point-cloud data to show that our representation leads to superior predictive accuracy over existing representations for arbitrary rotation targets. Second, we use image data collected onboard ground and aerial vehicles to demonstrate that our representation is amenable to an effective out-of-distribution (OOD) rejection technique that significantly improves the robustness of rotation estimates to unseen environmental effects and corrupted input images, without requiring the use of an explicit likelihood loss, stochastic sampling, or an auxiliary classifier. This capability is key for safety-critical applications where detecting novel inputs can prevent catastrophic failure of learned models.
Best Student Paper Award
-
Learned Adjustment of Camera Gain and Exposure Time for Improved Visual Feature Detection and Matching
J. L. Tomasi
Master Thesis , University of Toronto, Toronto, Ontario, Canada, 2020. Bibtex |
Abstract @mastersthesis{2020_Tomasi_Learned_A,
abstract = {Ensuring that captured images contain useful information is paramount to successful visual navigation. In this thesis, we explore a data-driven approach to account for environmental lighting changes, improving the quality of images for use in visual odometry (VO). We investigate what qualities of an image are desirable for navigation through an empirical analysis of the outputs of the VO front end. Based on this analysis, we build and train a deep convolutional neural network model to predictively adjust camera gain and exposure time parameters such that consecutive images contain a maximal number of matchable features. Our training method leverages several novel datasets consisting of images captured with varied gain and exposure time settings in diverse environments. Through real-world experiments, we demonstrate that our network is able to anticipate and compensate for lighting changes and maintain a higher number of inlier feature matches compared with competing camera parameter control algorithms.},
address = {Toronto, Ontario, Canada},
author = {Justin Louis Tomasi},
month = {September},
school = {University of Toronto},
title = {Learned Adjustment of Camera Gain and Exposure Time for Improved Visual Feature Detection and Matching},
year = {2020}
}
Ensuring that captured images contain useful information is paramount to successful visual navigation. In this thesis, we explore a data-driven approach to account for environmental lighting changes, improving the quality of images for use in visual odometry (VO). We investigate what qualities of an image are desirable for navigation through an empirical analysis of the outputs of the VO front end. Based on this analysis, we build and train a deep convolutional neural network model to predictively adjust camera gain and exposure time parameters such that consecutive images contain a maximal number of matchable features. Our training method leverages several novel datasets consisting of images captured with varied gain and exposure time settings in diverse environments. Through real-world experiments, we demonstrate that our network is able to anticipate and compensate for lighting changes and maintain a higher number of inlier feature matches compared with competing camera parameter control algorithms.
-
Learned Camera Gain and Exposure Control for Improved Visual Feature Detection and Matching
J. Tomasi, B. Wagstaff, S. Waslander, and J. Kelly
IEEE Robotics and Automation Letters, 2020. Bibtex |
Abstract @article{2020_Tomasi_Learned_B,
abstract = {Ensuring that captured images contain useful information is critical for successful visual navigation. In this paper, we explore a data-driven approach to account for environmental lighting changes, improving the quality of images for use in visual odometry (VO) or visual simultaneous localization and mapping (SLAM). We train a deep convolutional neural network model to predictively adjust camera gain and exposure time parameters such that consecutive images contain a maximal number of matchable features. The training process is fully self-supervised: our training signal is derived from an underlying VO or SLAM pipeline and, as a result, the model is optimized to perform well with that specific pipeline. We demonstrate through extensive real-world experiments that our network can anticipate and compensate for dramatic lighting changes (e.g., transitions into and out of road tunnels), maintaining a substantially higher number of inlier feature matches than competing camera parameter control algorithms.},
author = {Justin Tomasi and Brandon Wagstaff and Steven Waslander and Jonathan Kelly},
journal = {{IEEE} Robotics and Automation Letters},
note = {Submitted},
title = {Learned Camera Gain and Exposure Control for Improved Visual Feature Detection and Matching},
year = {2020}
}
Ensuring that captured images contain useful information is critical for successful visual navigation. In this paper, we explore a data-driven approach to account for environmental lighting changes, improving the quality of images for use in visual odometry (VO) or visual simultaneous localization and mapping (SLAM). We train a deep convolutional neural network model to predictively adjust camera gain and exposure time parameters such that consecutive images contain a maximal number of matchable features. The training process is fully self-supervised: our training signal is derived from an underlying VO or SLAM pipeline and, as a result, the model is optimized to perform well with that specific pipeline. We demonstrate through extensive real-world experiments that our network can anticipate and compensate for dramatic lighting changes (e.g., transitions into and out of road tunnels), maintaining a substantially higher number of inlier feature matches than competing camera parameter control algorithms.
Submitted
-
Robust Data-Driven Zero-Velocity Detection for Foot-Mounted Inertial Navigation
B. Wagstaff, V. Peretroukhin, and J. Kelly
IEEE Sensors Journal, vol. 20, iss. 2, pp. 957-967, 2020. DOI |
Bibtex |
Abstract |
arXiv |
Code @article{2020_Wagstaff_Robust,
abstract = {We present two novel techniques for detecting zero-velocity events to improve foot-mounted inertial navigation. Our first technique augments a classical zero-velocity detector by incorporating a motion classifier that adaptively updates the detector's threshold parameter. Our second technique uses a long short-term memory (LSTM) recurrent neural network to classify zero-velocity events from raw inertial data, in contrast to the majority of zero-velocity detection methods that rely on basic statistical hypothesis testing. We demonstrate that both of our proposed detectors achieve higher accuracies than existing detectors for trajectories including walking, running, and stair-climbing motions. Additionally, we present a straightforward data augmentation method that is able to extend the LSTM-based model to different inertial sensors without the need to collect new training data.},
author = {Brandon Wagstaff and Valentin Peretroukhin and Jonathan Kelly},
code = {https://github.com/utiasSTARS/pyshoe},
doi = {10.1109/JSEN.2019.2944412},
journal = {{IEEE} Sensors Journal},
month = {January},
number = {2},
pages = {957--967},
title = {Robust Data-Driven Zero-Velocity Detection for Foot-Mounted Inertial Navigation},
url = {http://arxiv.org/abs/1910.00529},
volume = {20},
year = {2020}
}
We present two novel techniques for detecting zero-velocity events to improve foot-mounted inertial navigation. Our first technique augments a classical zero-velocity detector by incorporating a motion classifier that adaptively updates the detector's threshold parameter. Our second technique uses a long short-term memory (LSTM) recurrent neural network to classify zero-velocity events from raw inertial data, in contrast to the majority of zero-velocity detection methods that rely on basic statistical hypothesis testing. We demonstrate that both of our proposed detectors achieve higher accuracies than existing detectors for trajectories including walking, running, and stair-climbing motions. Additionally, we present a straightforward data augmentation method that is able to extend the LSTM-based model to different inertial sensors without the need to collect new training data.
-
Self-Supervised Deep Pose Corrections for Robust Visual Odometry
B. Wagstaff, V. Peretroukhin, and J. Kelly
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’20), Paris, France, May 31–Jun. 4, 2020, pp. 2331-2337. DOI |
Bibtex |
Abstract |
arXiv |
Video |
Code @inproceedings{2020_Wagstaff_Self-Supervised,
abstract = {We present a self-supervised deep pose correction (DPC) network that applies pose corrections to a visual odometry estimator to improve its accuracy. Instead of regressing inter-frame pose changes directly, we build on prior work that uses data-driven learning to regress pose corrections that account for systematic errors due to violations of modelling assumptions. Our self-supervised formulation removes any requirement for six-degrees-of-freedom ground truth and, in contrast to expectations, often improves overall navigation accuracy compared to a supervised approach. Through extensive experiments, we show that our self-supervised DPC network can significantly enhance the performance of classical monocular and stereo odometry estimators and substantially out-performs state-of-the-art learning-only approaches.},
address = {Paris, France},
author = {Brandon Wagstaff and Valentin Peretroukhin and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'20})},
code = {https://github.com/utiasSTARS/ss-dpc-net},
date = {2020-05-31/2020-06-04},
doi = {10.1109/ICRA40945.2020.9197562},
month = {May 31--Jun. 4},
pages = {2331--2337},
title = {Self-Supervised Deep Pose Corrections for Robust Visual Odometry},
url = {https://arxiv.org/abs/2002.12339},
video1 = {https://www.youtube.com/watch?v=AvNBUK4lTMo},
year = {2020}
}
We present a self-supervised deep pose correction (DPC) network that applies pose corrections to a visual odometry estimator to improve its accuracy. Instead of regressing inter-frame pose changes directly, we build on prior work that uses data-driven learning to regress pose corrections that account for systematic errors due to violations of modelling assumptions. Our self-supervised formulation removes any requirement for six-degrees-of-freedom ground truth and, in contrast to expectations, often improves overall navigation accuracy compared to a supervised approach. Through extensive experiments, we show that our self-supervised DPC network can significantly enhance the performance of classical monocular and stereo odometry estimators and substantially out-performs state-of-the-art learning-only approaches.
-
Certifiably Optimal Monocular Hand-Eye Calibration
E. Wise, M. Giamou, S. Khoubyarian, A. Grover, and J. Kelly
Proceedings of the IEEE International Conference on Multisensor Fusion and Integration for Intelligent Systems (MFI’20), Karlsruhe, Germany, Sep. 14–16, 2020. DOI |
Bibtex |
Abstract |
arXiv |
Video @inproceedings{2020_Wise_Certifiably,
abstract = {Correct fusion of data from two sensors requires an accurate estimate of their relative pose, which can be determined through the process of extrinsic calibration. When the sensors are capable of producing their own egomotion estimates (i.e., measurements of their trajectories through an environment), the `hand-eye' formulation of extrinsic calibration can be employed. In this paper, we extend our recent work on a convex optimization approach for hand-eye calibration to the case where one of the sensors cannot observe the scale of its translational motion (e.g., a monocular camera observing an unmapped environment). We prove that our technique is able to provide a certifiably globally optimal solution to both the known- and unknown-scale variants of hand-eye calibration, provided that the measurement noise is bounded. Herein, we focus on the theoretical aspects of the problem, show the tightness and stability of our convex relaxation, and demonstrate the optimality and speed of our algorithm through experiments with synthetic data.},
address = {Karlsruhe, Germany},
author = {Emmett Wise and Matthew Giamou and Soroush Khoubyarian and Abhinav Grover and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE} International Conference on Multisensor Fusion and Integration for Intelligent Systems {(MFI'20)}},
date = {2020-09-14/2020-09-16},
doi = {10.1109/MFI49285.2020.9235219},
month = {Sep. 14--16},
title = {Certifiably Optimal Monocular Hand-Eye Calibration},
url = {https://arxiv.org/abs/2005.08298},
video1 = {https://youtu.be/BdjGBvuaqVo},
year = {2020}
}
Correct fusion of data from two sensors requires an accurate estimate of their relative pose, which can be determined through the process of extrinsic calibration. When the sensors are capable of producing their own egomotion estimates (i.e., measurements of their trajectories through an environment), the `hand-eye' formulation of extrinsic calibration can be employed. In this paper, we extend our recent work on a convex optimization approach for hand-eye calibration to the case where one of the sensors cannot observe the scale of its translational motion (e.g., a monocular camera observing an unmapped environment). We prove that our technique is able to provide a certifiably globally optimal solution to both the known- and unknown-scale variants of hand-eye calibration, provided that the measurement noise is bounded. Herein, we focus on the theoretical aspects of the problem, show the tightness and stability of our convex relaxation, and demonstrate the optimality and speed of our algorithm through experiments with synthetic data.
-
Self-Supervised Scale Recovery for Monocular Depth and Egomotion Estimation
B. Wagstaff and J. Kelly
IEEE Robotics and Automation Letters, 2020. Bibtex |
Abstract |
arXiv @article{2021_Wagstaff_Self-Supervised,
abstract = {The self-supervised loss formulation for jointly training depth and egomotion neural networks with monocular images is well studied and has demonstrated state-of-the-art accuracy. One of the main limitations of this approach, however, is that the depth and egomotion estimates are only determined up to an unknown scale. In this paper, we present a novel scale recovery loss that enforces consistency between a known camera height and the estimated camera height, generating metric (scaled) depth and egomotion predictions. We show that our proposed method is competitive with other scale recovery techniques (i.e., pose supervision and stereo left/right consistency constraints). Further, we demonstrate how our method facilitates network retraining within new environments, whereas other scale-resolving approaches are incapable of doing so. Notably, our egomotion network is able to produce more accurate estimates than a similar method that only recovers scale at test time.},
author = {Brandon Wagstaff and Jonathan Kelly},
journal = {{IEEE} Robotics and Automation Letters},
note = {Submitted},
title = {Self-Supervised Scale Recovery for Monocular Depth and Egomotion Estimation},
url = {https://arxiv.org/abs/2009.03787},
year = {2020}
}
The self-supervised loss formulation for jointly training depth and egomotion neural networks with monocular images is well studied and has demonstrated state-of-the-art accuracy. One of the main limitations of this approach, however, is that the depth and egomotion estimates are only determined up to an unknown scale. In this paper, we present a novel scale recovery loss that enforces consistency between a known camera height and the estimated camera height, generating metric (scaled) depth and egomotion predictions. We show that our proposed method is competitive with other scale recovery techniques (i.e., pose supervision and stereo left/right consistency constraints). Further, we demonstrate how our method facilitates network retraining within new environments, whereas other scale-resolving approaches are incapable of doing so. Notably, our egomotion network is able to produce more accurate estimates than a similar method that only recovers scale at test time.
Submitted
-
Matchable Image Transformations for Long-term Metric Visual Localization
L. Clement, M. Gridseth, J. Tomasi, and J. Kelly
Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR’19) Workshop on Image Matching: Local Features & Beyond, Long Beach, California, USA, Jun. 16–20, 2019. Bibtex |
PDF @inproceedings{2019_Clement_Matchable,
address = {Long Beach, California, USA},
author = {Lee Clement and Mona Gridseth and Justin Tomasi and Jonathan Kelly},
booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR'19) Workshop on Image Matching: Local Features \& Beyond},
date = {2019-06-16/2019-06-20},
month = {Jun. 16--20},
title = {Matchable Image Transformations for Long-term Metric Visual Localization},
year = {2019}
}
-
Where Do We Go From Here? Debates on the Future of Robotics Research at ICRA 2019
L. Clement, V. Peretroukhin, M. Giamou, J. Leonard, H. Kress-Gazit, J. How, M. Milford, O. Brock, R. Gariepy, A. P. Schoellig, N. Roy, H. Siegel, L. Righetti, A. Billard, and J. Kelly
IEEE Robotics & Automation Magazine, vol. 26, iss. 3, pp. 7-10, 2019. DOI |
Bibtex |
PDF @article{2019_Clement_Where,
author = {Lee Clement and Valentin Peretroukhin and Matthew Giamou and John Leonard and Hadas Kress-Gazit and Jonathan How and Michael Milford and Oliver Brock and Ryan Gariepy and Angela P. Schoellig and Nicholas Roy and Hallie Siegel and Ludovic Righetti and Aude Billard and Jonathan Kelly},
doi = {10.1109/MRA.2019.2926934},
journal = {{IEEE} Robotics \& Automation Magazine},
month = {September},
number = {3},
pages = {7--10},
title = {Where Do We Go From Here? Debates on the Future of Robotics Research at ICRA 2019},
volume = {26},
year = {2019}
}
-
Certifiably Globally Optimal Extrinsic Calibration from Per-Sensor Egomotion
M. Giamou, Z. Ma, V. Peretroukhin, and J. Kelly
IEEE Robotics and Automation Letters, vol. 4, iss. 2, pp. 367-374, 2019. DOI |
Bibtex |
Abstract |
arXiv |
Code @article{2019_Giamou_Certifiably,
abstract = {We present a certifiably globally optimal algorithm for determining the extrinsic calibration between two sensors that are capable of producing independent egomotion estimates. This problem has been previously solved using a variety of techniques, including local optimization approaches that have no formal global optimality guarantees. We use a quadratic objective function to formulate calibration as a quadratically constrained quadratic program (QCQP). By leveraging recent advances in the optimization of QCQPs, we are able to use existing semidefinite program (SDP) solvers to obtain a certifiably global optimum via the Lagrangian dual problem. Our problem formulation can be globally optimized by existing general-purpose solvers in less than a second, regardless of the number of measurements available and the noise level. This enables a variety of robotic platforms to rapidly and robustly compute and certify a globally optimal set of calibration parameters without a prior estimate or operator intervention. We compare the performance of our approach with a local solver on extensive simulations and multiple real datasets. Finally, we present necessary observability conditions that connect our approach to recent theoretical results and analytically support the empirical performance of our system.},
author = {Matthew Giamou and Ziye Ma and Valentin Peretroukhin and Jonathan Kelly},
code = {https://github.com/utiasSTARS/certifiable-calibration},
doi = {10.1109/LRA.2018.2890444},
journal = {{IEEE} Robotics and Automation Letters},
month = {April},
number = {2},
pages = {367--374},
title = {Certifiably Globally Optimal Extrinsic Calibration from Per-Sensor Egomotion},
url = {https://arxiv.org/abs/1809.03554},
volume = {4},
year = {2019}
}
We present a certifiably globally optimal algorithm for determining the extrinsic calibration between two sensors that are capable of producing independent egomotion estimates. This problem has been previously solved using a variety of techniques, including local optimization approaches that have no formal global optimality guarantees. We use a quadratic objective function to formulate calibration as a quadratically constrained quadratic program (QCQP). By leveraging recent advances in the optimization of QCQPs, we are able to use existing semidefinite program (SDP) solvers to obtain a certifiably global optimum via the Lagrangian dual problem. Our problem formulation can be globally optimized by existing general-purpose solvers in less than a second, regardless of the number of measurements available and the noise level. This enables a variety of robotic platforms to rapidly and robustly compute and certify a globally optimal set of calibration parameters without a prior estimate or operator intervention. We compare the performance of our approach with a local solver on extensive simulations and multiple real datasets. Finally, we present necessary observability conditions that connect our approach to recent theoretical results and analytically support the empirical performance of our system.
-
Leveraging Robotics Education to Improve Prosperity in Developing Nations: An Early Case Study in Myanmar
J. Kelly, H. Htet, and J. Dutra
Proceedings of the Do Good Robotics Symposium (DGRS’19), College Park, Maryland, USA, Oct. 3–4, 2019. Bibtex |
Abstract |
PDF @inproceedings{2019_Kelly_Leveraging,
abstract = {Robotics can be a powerful educational tool: the topic is exciting, timely, and highly engaging. Research has shown that robotics courses can drive students' interest in science, technology, engineering, and mathematics (STEM) careers. While many successful outreach and introductory programs exist in developed countries, an open question is how best to leverage the appeal of robotics to improve educational outcomes (and, ultimately, prosperity) in developing countries. What material is most relevant? How should that material be presented to engage with students? And how do we measure the impact of such initiatives? In this paper, we report on the design and delivery of a short course on self-driving vehicles for a group of students in the developing nation of Myanmar. The pilot program was facilitated through cooperation with Phandeeyar, a unique innovation hub and startup accelerator based in Yangon. We discuss the motivation for the program, the choice of topic, and the student experience. We close by offering some preliminary thoughts about quantifying the value of this type of robotics outreach effort and of robotics education, both in Myanmar and beyond.},
address = {College Park, Maryland, USA},
author = {Jonathan Kelly and Htoo Htet and Joao Dutra},
booktitle = {Proceedings of the Do Good Robotics Symposium (DGRS'19)},
date = {2019-10-03/2019-10-04},
month = {Oct. 3--4},
title = {Leveraging Robotics Education to Improve Prosperity in Developing Nations: An Early Case Study in Myanmar},
year = {2019}
}
Robotics can be a powerful educational tool: the topic is exciting, timely, and highly engaging. Research has shown that robotics courses can drive students' interest in science, technology, engineering, and mathematics (STEM) careers. While many successful outreach and introductory programs exist in developed countries, an open question is how best to leverage the appeal of robotics to improve educational outcomes (and, ultimately, prosperity) in developing countries. What material is most relevant? How should that material be presented to engage with students? And how do we measure the impact of such initiatives? In this paper, we report on the design and delivery of a short course on self-driving vehicles for a group of students in the developing nation of Myanmar. The pilot program was facilitated through cooperation with Phandeeyar, a unique innovation hub and startup accelerator based in Yangon. We discuss the motivation for the program, the choice of topic, and the student experience. We close by offering some preliminary thoughts about quantifying the value of this type of robotics outreach effort and of robotics education, both in Myanmar and beyond.
-
Fast Manipulability Maximization Using Continuous-Time Trajectory Optimization
F. Maric, O. Limoyo, L. Petrovic, T. Ablett, I. Petrovic, and J. Kelly
Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS’19), Macau, China, Nov. 4–8, 2019, pp. 8258-8264. DOI |
Bibtex |
Abstract |
arXiv |
Video @inproceedings{2019_Maric_Fast,
abstract = {A significant challenge in manipulation motion planning is to ensure agility in the face of unpredictable changes during task execution. This requires the identification and possible modification of suitable joint-space trajectories, since the joint velocities required to achieve a specific end-effector motion vary with manipulator configuration. For a given manipulator configuration, the joint space-to-task space velocity mapping is characterized by a quantity known as the manipulability index. In contrast to previous control-based approaches, we examine the maximization of manipulability during planning as a way of achieving adaptable and safe joint space-to-task space motion mappings in various scenarios. By representing the manipulator trajectory as a continuous-time Gaussian process (GP), we are able to leverage recent advances in trajectory optimization to maximize the manipulability index during trajectory generation. Moreover, the sparsity of our chosen representation reduces the typically large computational cost associated with maximizing manipulability when additional constraints exist. Results from simulation studies and experiments with a real manipulator demonstrate increases in manipulability, while maintaining smooth trajectories with more dexterous (and therefore more agile) arm configurations.},
address = {Macau, China},
author = {Filip Maric and Oliver Limoyo and Luka Petrovic and Trevor Ablett and Ivan Petrovic and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE/RSJ} International Conference on Intelligent Robots and Systems {(IROS'19)}},
date = {2019-11-04/2019-11-08},
doi = {10.1109/IROS40897.2019.8968441},
month = {Nov. 4--8},
pages = {8258--8264},
title = {Fast Manipulability Maximization Using Continuous-Time Trajectory Optimization},
url = {https://arxiv.org/abs/1908.02963},
video1 = {https://www.youtube.com/watch?v=tB34VfDrF84},
year = {2019}
}
A significant challenge in manipulation motion planning is to ensure agility in the face of unpredictable changes during task execution. This requires the identification and possible modification of suitable joint-space trajectories, since the joint velocities required to achieve a specific end-effector motion vary with manipulator configuration. For a given manipulator configuration, the joint space-to-task space velocity mapping is characterized by a quantity known as the manipulability index. In contrast to previous control-based approaches, we examine the maximization of manipulability during planning as a way of achieving adaptable and safe joint space-to-task space motion mappings in various scenarios. By representing the manipulator trajectory as a continuous-time Gaussian process (GP), we are able to leverage recent advances in trajectory optimization to maximize the manipulability index during trajectory generation. Moreover, the sparsity of our chosen representation reduces the typically large computational cost associated with maximizing manipulability when additional constraints exist. Results from simulation studies and experiments with a real manipulator demonstrate increases in manipulability, while maintaining smooth trajectories with more dexterous (and therefore more agile) arm configurations.
-
An Observability Based Approach to Flight Path Reconstruction of Uninformative Coupled Aircraft Trajectories: A Case Study Considering Stall Maneuvers for Aircraft Certification
G. Moszczynski, M. Giamou, J. Leung, J. Kelly, and P. Grant
AIAA Science and Technology Forum and Exposition (AIAA SciTech), San Diego, California, USA, Jan. 7–11, 2019. Bibtex |
Abstract @inproceedings{2019_Moszczynski_Observability,
abstract = {Based on the demonstrated efficacy of observability metrics in the realm of informative trajectory optimization for sensor calibration, the application of such metrics within the context of flight path reconstruction is investigated. The minimum singular value of the observability Gramian is adopted to describe flight test information content, and used to mathematically characterize parameter estimation difficulties discussed throughout the body of literature on flight path reconstruction. A metric for total information content of a set of flight test experiments is then presented and used to motivate FPR based on multiple flight test experiments. A highly efficient maximum a posteriori trajectory estimation scheme accommodating the use of multiple flight test experiments is then presented. The finalization of this work will present the application of the adopted information metric and developed estimation scheme to a case study concerning reconstruction of stall maneuver data with poor information content collected for aircraft certification purposes.},
address = {San Diego, California, USA},
author = {Gregory Moszczynski and Matthew Giamou and Jordan Leung and Jonathan Kelly and Peter Grant},
booktitle = {{AIAA} Science and Technology Forum and Exposition {(AIAA SciTech)}},
date = {2019-01-07/2019-01-11},
month = {Jan. 7--11},
title = {An Observability Based Approach to Flight Path Reconstruction of Uninformative Coupled Aircraft Trajectories: A Case Study Considering Stall Maneuvers for Aircraft Certification},
year = {2019}
}
Based on the demonstrated efficacy of observability metrics in the realm of informative trajectory optimization for sensor calibration, the application of such metrics within the context of flight path reconstruction is investigated. The minimum singular value of the observability Gramian is adopted to describe flight test information content, and used to mathematically characterize parameter estimation difficulties discussed throughout the body of literature on flight path reconstruction. A metric for total information content of a set of flight test experiments is then presented and used to motivate FPR based on multiple flight test experiments. A highly efficient maximum a posteriori trajectory estimation scheme accommodating the use of multiple flight test experiments is then presented. The finalization of this work will present the application of the adopted information metric and developed estimation scheme to a case study concerning reconstruction of stall maneuver data with poor information content collected for aircraft certification purposes.
-
Deep Probabilistic Regression of Elements of SO(3) using Quaternion Averaging and Uncertainty Injection
V. Peretroukhin, B. Wagstaff, and J. Kelly
Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR’19) Workshop on Uncertainty and Robustness in Deep Visual Learning, Long Beach, California, USA, Jun. 16–20, 2019, pp. 83-86. Bibtex |
Abstract |
arXiv |
Code @inproceedings{2019_Peretroukhin_Deep,
abstract = {Consistent estimates of rotation are crucial to vision- based motion estimation in augmented reality and robotics. In this work, we present a method to extract probabilistic estimates of rotation from deep regression models. First, we build on prior work and develop a multi-headed network structure we name HydraNet that can account for both aleatoric and epistemic uncertainty. Second, we extend HydraNet to targets that belong to the rotation group, SO(3), by regressing unit quaternions and using the tools of rotation averaging and uncertainty injection onto the manifold to produce three-dimensional covariances. Finally, we present results and analysis on a synthetic dataset, learn consistent orientation estimates on the 7-Scenes dataset, and show how we can use our learned covariances to fuse deep estimates of relative orientation with classical stereo visual odometry to improve localization on the KITTI dataset.},
address = {Long Beach, California, USA},
author = {Valentin Peretroukhin and Brandon Wagstaff and Jonathan Kelly},
booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR'19) Workshop on Uncertainty and Robustness in Deep Visual Learning},
code = {https://github.com/utiasSTARS/so3_learning},
date = {2019-06-16/2019-06-20},
longurl = {http://openaccess.thecvf.com/content_CVPRW_2019/papers/Uncertainty%20and%20Robustness%20in%20Deep%20Visual%20Learning/Peretroukhin_Deep_Probabilistic_Regression_of_Elements_of_SO3_using_Quaternion_Averaging_CVPRW_2019_paper.pdf},
month = {Jun. 16--20},
pages = {83--86},
title = {Deep Probabilistic Regression of Elements of SO(3) using Quaternion Averaging and Uncertainty Injection},
url = {https://arxiv.org/abs/1904.03182},
year = {2019}
}
Consistent estimates of rotation are crucial to vision- based motion estimation in augmented reality and robotics. In this work, we present a method to extract probabilistic estimates of rotation from deep regression models. First, we build on prior work and develop a multi-headed network structure we name HydraNet that can account for both aleatoric and epistemic uncertainty. Second, we extend HydraNet to targets that belong to the rotation group, SO(3), by regressing unit quaternions and using the tools of rotation averaging and uncertainty injection onto the manifold to produce three-dimensional covariances. Finally, we present results and analysis on a synthetic dataset, learn consistent orientation estimates on the 7-Scenes dataset, and show how we can use our learned covariances to fuse deep estimates of relative orientation with classical stereo visual odometry to improve localization on the KITTI dataset.
-
The Phoenix Drone: An Open-Source Dual-Rotor Tail-Sitter Platform for Research and Education
Y. Wu, X. Du, R. Duivenvoorden, and J. Kelly
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’19), Montreal, Quebec, Canada, May 20–24, 2019, pp. 5330-5336. DOI |
Bibtex |
Abstract |
arXiv |
Video |
Code @inproceedings{2019_Wu_Phoenix,
abstract = {In this paper, we introduce the Phoenix drone: the first completely open-source tail-sitter micro aerial vehicle (MAV) platform. The vehicle has a highly versatile, dual-rotor design and is engineered to be low-cost and easily extensible/modifiable. Our open-source release includes all of the design documents, software resources, and simulation tools needed to build and fly a high-performance tail-sitter for research and educational purposes.
The drone has been developed for precision flight with a high degree of control authority. Our design methodology included extensive testing and characterization of the aerodynamic properties of the vehicle. The platform incorporates many off-the-shelf components and 3D-printed parts, in order to keep the cost down. Nonetheless, the paper includes results from flight trials which demonstrate that the vehicle is capable of very stable hovering and accurate trajectory tracking.
Our hope is that the open-source Phoenix reference design will be useful to both researchers and educators. In particular, the details in this paper and the available open-source materials should enable learners to gain an understanding of aerodynamics, flight control, state estimation, software design, and simulation, while experimenting with a unique aerial robot.},
address = {Montreal, Quebec, Canada},
author = {Yilun Wu and Xintong Du and Rikky Duivenvoorden and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'19})},
code = {https://github.com/utiasSTARS/PhoenixDrone},
date = {2019-05-20/2019-05-24},
doi = {10.1109/ICRA.2019.8794433},
month = {May 20--24},
pages = {5330--5336},
title = {The Phoenix Drone: An Open-Source Dual-Rotor Tail-Sitter Platform for Research and Education},
url = {https://arxiv.org/abs/1810.03196},
video1 = {https://www.youtube.com/watch?v=VSAk3Z0G08Q},
year = {2019}
}
In this paper, we introduce the Phoenix drone: the first completely open-source tail-sitter micro aerial vehicle (MAV) platform. The vehicle has a highly versatile, dual-rotor design and is engineered to be low-cost and easily extensible/modifiable. Our open-source release includes all of the design documents, software resources, and simulation tools needed to build and fly a high-performance tail-sitter for research and educational purposes. The drone has been developed for precision flight with a high degree of control authority. Our design methodology included extensive testing and characterization of the aerodynamic properties of the vehicle. The platform incorporates many off-the-shelf components and 3D-printed parts, in order to keep the cost down. Nonetheless, the paper includes results from flight trials which demonstrate that the vehicle is capable of very stable hovering and accurate trajectory tracking. Our hope is that the open-source Phoenix reference design will be useful to both researchers and educators. In particular, the details in this paper and the available open-source materials should enable learners to gain an understanding of aerodynamics, flight control, state estimation, software design, and simulation, while experimenting with a unique aerial robot.
-
How to Train a CAT: Learning Canonical Appearance Transformations for Robust Direct Localization Under Illumination Change
L. Clement and J. Kelly
IEEE Robotics and Automation Letters, vol. 3, iss. 3, pp. 2447-2454, 2018. DOI |
Bibtex |
Abstract |
arXiv |
Video |
Code @article{2018_Clement_Learning,
abstract = {Direct visual localization has recently enjoyed a resurgence in popularity with the increasing availability of cheap mobile computing power. The competitive accuracy and robustness of these algorithms compared to state-of-the-art feature-based methods, as well as their natural ability to yield dense maps, makes them an appealing choice for a variety of mobile robotics applications. However, direct methods remain brittle in the face of appearance change due to their underlying assumption of photometric consistency, which is commonly violated in practice. In this paper, we propose to mitigate this problem by training deep convolutional encoder-decoder models to transform images of a scene such that they correspond to a chosen canonical appearance such as static diffuse illumination. We validate our method in multiple environments and illumination conditions using high-fidelity synthetic RGB-D datasets, and integrate the trained models into a direct visual localization pipeline, yielding improvements in visual odometry (VO) accuracy through time-varying illumination conditions, as well as improved relocalization performance under illumination change, where conventional methods normally fail.},
author = {Lee Clement and Jonathan Kelly},
code = {https://github.com/utiasSTARS/cat-net},
doi = {10.1109/LRA.2018.2799741},
journal = {{IEEE} Robotics and Automation Letters},
month = {July},
number = {3},
pages = {2447--2454},
title = {How to Train a {CAT}: Learning Canonical Appearance Transformations for Robust Direct Localization Under Illumination Change},
url = {https://arxiv.org/abs/1709.03009},
video1 = {https://www.youtube.com/watch?v=ej6VNBq3dDE},
volume = {3},
year = {2018}
}
Direct visual localization has recently enjoyed a resurgence in popularity with the increasing availability of cheap mobile computing power. The competitive accuracy and robustness of these algorithms compared to state-of-the-art feature-based methods, as well as their natural ability to yield dense maps, makes them an appealing choice for a variety of mobile robotics applications. However, direct methods remain brittle in the face of appearance change due to their underlying assumption of photometric consistency, which is commonly violated in practice. In this paper, we propose to mitigate this problem by training deep convolutional encoder-decoder models to transform images of a scene such that they correspond to a chosen canonical appearance such as static diffuse illumination. We validate our method in multiple environments and illumination conditions using high-fidelity synthetic RGB-D datasets, and integrate the trained models into a direct visual localization pipeline, yielding improvements in visual odometry (VO) accuracy through time-varying illumination conditions, as well as improved relocalization performance under illumination change, where conventional methods normally fail.
-
Overcoming the Challenges of Solar Rover Autonomy: Enabling Long-Duration Planetary Navigation
O. Lamarre and J. Kelly
Proceedings of the 14th International Symposium on Artificial Intelligence, Robotics and Automation in Space (i-SAIRAS’18), Madrid, Spain, Jun. 4–6, 2018. Bibtex |
Abstract |
arXiv @inproceedings{2018_Lamarre_Overcoming,
abstract = {The successes of previous and current Mars rovers have encouraged space agencies worldwide to pursue additional planetary exploration missions with more ambitious navigation goals. For example, NASA's planned Mars Sample Return mission will be a multi-year undertaking that will require a solar-powered rover to drive over 150 metres per sol for approximately three months. This paper reviews the mobility planning framework used by current rovers and surveys the major challenges involved in continuous long-distance navigation on the Red Planet. It also discusses recent work related to environment-aware and energy-aware navigation, and provides a perspective on how such work may eventually allow a solar-powered rover to achieve autonomous long-distance navigation on Mars.},
address = {Madrid, Spain},
author = {Olivier Lamarre and Jonathan Kelly},
booktitle = {Proceedings of the 14th International Symposium on Artificial Intelligence, Robotics and Automation in Space {(i-SAIRAS'18)}},
date = {2018-06-04/2018-06-06},
month = {Jun. 4--6},
title = {Overcoming the Challenges of Solar Rover Autonomy: Enabling Long-Duration Planetary Navigation},
url = {https://arxiv.org/abs/1805.05451},
year = {2018}
}
The successes of previous and current Mars rovers have encouraged space agencies worldwide to pursue additional planetary exploration missions with more ambitious navigation goals. For example, NASA's planned Mars Sample Return mission will be a multi-year undertaking that will require a solar-powered rover to drive over 150 metres per sol for approximately three months. This paper reviews the mobility planning framework used by current rovers and surveys the major challenges involved in continuous long-distance navigation on the Red Planet. It also discusses recent work related to environment-aware and energy-aware navigation, and provides a perspective on how such work may eventually allow a solar-powered rover to achieve autonomous long-distance navigation on Mars.
-
Self-Calibration of Mobile Manipulator Kinematic and Sensor Extrinsic Parameters Through Contact-Based Interaction
O. Limoyo, T. Ablett, F. Maric, L. Volpatti, and J. Kelly
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’18), Brisbane, Queensland, Australia, May 21–25, 2018. DOI |
Bibtex |
Abstract |
arXiv |
Video @inproceedings{2018_Limoyo_Self-Calibration,
abstract = {We present a novel approach for mobile manipulator self-calibration using contact information. Our method, based on point cloud registration, is applied to estimate the extrinsic transform between a fixed vision sensor mounted on a mobile base and an end effector. Beyond sensor calibration, we demonstrate that the method can be extended to include manipulator kinematic model parameters, which involves a non-rigid registration process. Our procedure uses on-board sensing exclusively and does not rely on any external measurement devices, fiducial markers, or calibration rigs. Further, it is fully automatic in the general case. We experimentally validate the proposed method on a custom mobile manipulator platform, and demonstrate centimetre-level post-calibration accuracy in positioning of the end effector using visual guidance only. We also discuss the stability properties of the registration algorithm, in order to determine the conditions under which calibration is possible.},
address = {Brisbane, Queensland, Australia},
author = {Oliver Limoyo and Trevor Ablett and Filip Maric and Luke Volpatti and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'18})},
date = {2018-05-21/2018-05-25},
doi = {10.1109/ICRA.2018.8460658},
month = {May 21--25},
title = {Self-Calibration of Mobile Manipulator Kinematic and Sensor Extrinsic Parameters Through Contact-Based Interaction},
url = {https://arxiv.org/abs/1803.06406},
video1 = {https://www.youtube.com/watch?v=cz9UB-BcGA0},
year = {2018}
}
We present a novel approach for mobile manipulator self-calibration using contact information. Our method, based on point cloud registration, is applied to estimate the extrinsic transform between a fixed vision sensor mounted on a mobile base and an end effector. Beyond sensor calibration, we demonstrate that the method can be extended to include manipulator kinematic model parameters, which involves a non-rigid registration process. Our procedure uses on-board sensing exclusively and does not rely on any external measurement devices, fiducial markers, or calibration rigs. Further, it is fully automatic in the general case. We experimentally validate the proposed method on a custom mobile manipulator platform, and demonstrate centimetre-level post-calibration accuracy in positioning of the end effector using visual guidance only. We also discuss the stability properties of the registration algorithm, in order to determine the conditions under which calibration is possible.
-
Manipulability Maximization Using Continuous-Time Gaussian Processes
F. Maric, O. Limoyo, L. Petrovic, I. Petrovic, and J. Kelly
Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS’18) Workshop Towards Robots that Exhibit Manipulation Intelligence, Madrid, Spain, Oct. 1, 2018. Bibtex |
Abstract |
arXiv @inproceedings{2018_Maric_Manipulabiility,
abstract = {A significant challenge in motion planning is to avoid being in or near singular configurations (singularities), that is, joint configurations that result in the loss of the ability to move in certain directions in task space. A robotic system's capacity for motion is reduced even in regions that are in close proximity to (i.e., neighbouring) a singularity. In this work we examine singularity avoidance in a motion planning context, finding trajectories which minimize proximity to singular regions, subject to constraints. We define a manipulability-based likelihood associated with singularity avoidance over a continuous trajectory representation, which we then maximize using a maximum a posteriori (MAP) estimator. Viewing the MAP problem as inference on a factor graph, we use gradient information from interpolated states to maximize the trajectory's overall manipulability. Both qualitative and quantitative analyses of experimental data show increases in manipulability that result in smooth trajectories with visibly more dexterous arm configurations.},
address = {Madrid, Spain},
author = {Filip Maric and Oliver Limoyo and Luka Petrovic and Ivan Petrovic and Jonathan Kelly},
booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS'18) Workshop Towards Robots that Exhibit Manipulation Intelligence},
date = {2018-10-01},
month = {Oct. 1},
title = {Manipulability Maximization Using Continuous-Time Gaussian Processes},
url = {https://arxiv.org/abs/1803.09493},
year = {2018}
}
A significant challenge in motion planning is to avoid being in or near singular configurations (singularities), that is, joint configurations that result in the loss of the ability to move in certain directions in task space. A robotic system's capacity for motion is reduced even in regions that are in close proximity to (i.e., neighbouring) a singularity. In this work we examine singularity avoidance in a motion planning context, finding trajectories which minimize proximity to singular regions, subject to constraints. We define a manipulability-based likelihood associated with singularity avoidance over a continuous trajectory representation, which we then maximize using a maximum a posteriori (MAP) estimator. Viewing the MAP problem as inference on a factor graph, we use gradient information from interpolated states to maximize the trajectory's overall manipulability. Both qualitative and quantitative analyses of experimental data show increases in manipulability that result in smooth trajectories with visibly more dexterous arm configurations.
-
Unified Spatiotemporal Calibration of Egomotion Sensors and 2D Lidars in Arbitrary Environments
J. Marr
Master Thesis , University of Toronto, Toronto, Ontario, Canada, 2018. Bibtex |
Abstract @mastersthesis{2018_Marr_Unified,
abstract = {This thesis aims to develop an automatic spatiotemporal calibration routine for lidars and egomotion sensors that relaxes many common requirements, such as the need for overlapping sensor fields of view, or calibration targets with known dimensions. In particular, a set of entropy-based calibration algorithms are extended to allow estimation of sensor clock time offsets in tandem with sensor-to-sensor spatial transformations. A novel Bayesian optimization routine is developed to address the non-smooth behaviour observed in the entropy cost function at small scales. The routine is tested on both simulation and real world data. Simulation results show that, given a set of lidar data taken from many different viewpoints, the calibration can be constrained to within less than 5 mm, 0.1 degrees, and 0.15 ms in the translational, rotational, and time-delay parameters respectively. For real-world data, in the absence of a reliable ground truth, we present results that show a repeatability of $\pm$ 4 mm, 1 degree, and 0.1 ms. When a monocular camera is used as the egomotion sensor, the routine is able to resolve the scale of the trajectory. A very brief analysis of the applicability of the method to Inertial Measurement Unit (IMU) to lidar calibration is presented.},
address = {Toronto, Ontario, Canada},
author = {Jordan Marr},
month = {September},
school = {University of Toronto},
title = {Unified Spatiotemporal Calibration of Egomotion Sensors and 2D Lidars in Arbitrary Environments},
year = {2018}
}
This thesis aims to develop an automatic spatiotemporal calibration routine for lidars and egomotion sensors that relaxes many common requirements, such as the need for overlapping sensor fields of view, or calibration targets with known dimensions. In particular, a set of entropy-based calibration algorithms are extended to allow estimation of sensor clock time offsets in tandem with sensor-to-sensor spatial transformations. A novel Bayesian optimization routine is developed to address the non-smooth behaviour observed in the entropy cost function at small scales. The routine is tested on both simulation and real world data. Simulation results show that, given a set of lidar data taken from many different viewpoints, the calibration can be constrained to within less than 5 mm, 0.1 degrees, and 0.15 ms in the translational, rotational, and time-delay parameters respectively. For real-world data, in the absence of a reliable ground truth, we present results that show a repeatability of $\pm$ 4 mm, 1 degree, and 0.1 ms. When a monocular camera is used as the egomotion sensor, the routine is able to resolve the scale of the trajectory. A very brief analysis of the applicability of the method to Inertial Measurement Unit (IMU) to lidar calibration is presented.
-
DPC-Net: Deep Pose Correction for Visual Localization
V. Peretroukhin and J. Kelly
IEEE Robotics and Automation Letters, vol. 3, iss. 3, pp. 2424-2431, 2018. DOI |
Bibtex |
Abstract |
arXiv |
Video |
Code @article{2018_Peretroukhin_Deep,
abstract = {We present a novel method to fuse the power of deep networks with the computational efficiency of geometric and probabilistic localization algorithms. In contrast to other methods that completely replace a classical visual estimator with a deep network, we propose an approach that uses a convolutional neural network to learn difficult-to-model corrections to the estimator from ground-truth training data. To this end, we derive a novel loss function for learning SE{3} corrections based on a matrix Lie groups approach, with a natural formulation for balancing translation and rotation errors. We use this loss to train a Deep Pose Correction network (DPC-Net) that learns to predict corrections for a particular estimator, sensor and environment. Using the KITTI odometry dataset, we demonstrate significant improvements to the accuracy of a computationally-efficient sparse stereo visual odometry pipeline, that render it as accurate as a modern computationally-intensive dense estimator. Further, we show how DPC-Net can be used to mitigate the effect of poorly calibrated lens distortion parameters.},
author = {Valentin Peretroukhin and Jonathan Kelly},
code = {https://github.com/utiasSTARS/dpc-net},
doi = {10.1109/LRA.2017.2778765},
journal = {{IEEE} Robotics and Automation Letters},
month = {July},
number = {3},
pages = {2424--2431},
title = {{DPC-Net}: Deep Pose Correction for Visual Localization},
url = {https://arxiv.org/abs/1709.03128},
video1 = {https://www.youtube.com/watch?v=j9jnLldUAkc},
volume = {3},
year = {2018}
}
We present a novel method to fuse the power of deep networks with the computational efficiency of geometric and probabilistic localization algorithms. In contrast to other methods that completely replace a classical visual estimator with a deep network, we propose an approach that uses a convolutional neural network to learn difficult-to-model corrections to the estimator from ground-truth training data. To this end, we derive a novel loss function for learning SE3 corrections based on a matrix Lie groups approach, with a natural formulation for balancing translation and rotation errors. We use this loss to train a Deep Pose Correction network (DPC-Net) that learns to predict corrections for a particular estimator, sensor and environment. Using the KITTI odometry dataset, we demonstrate significant improvements to the accuracy of a computationally-efficient sparse stereo visual odometry pipeline, that render it as accurate as a modern computationally-intensive dense estimator. Further, we show how DPC-Net can be used to mitigate the effect of poorly calibrated lens distortion parameters.
-
Inferring Sun Direction to Improve Visual Odometry: A Deep Learning Approach
V. Peretroukhin, L. Clement, and J. Kelly
The International Journal of Robotics Research, vol. 37, iss. 9, pp. 996-1016, 2018. DOI |
Bibtex |
Abstract |
Code @article{2018_Peretroukhin_Inferring,
abstract = {We present a method to incorporate global orientation information from the sun into a visual odometry pipeline using only the existing image stream, in which the sun is typically not visible. We leverage recent advances in Bayesian convolutional neural networks (BCNNs) to train and implement a sun detection model (dubbed Sun-BCNN) that infers a 3D sun direction vector from a single RGB image. Crucially, our method also computes a principled uncertainty associated with each prediction, using a Monte Carlo dropout scheme. We incorporate this uncertainty into a sliding window stereo visual odometry pipeline where accurate uncertainty estimates are critical for optimal data fusion. We evaluate our method on 21.6 km of urban driving data from the KITTI odometry benchmark where it achieves a median error of approximately 12 degrees and yields improvements of up to 42\% in translational average root mean squared error (ARMSE) and 32\% in rotational ARMSE compared with standard visual odometry. We further evaluate our method on an additional 10 km of visual navigation data from the Devon Island Rover Navigation dataset, achieving a median error of less than 8 degrees and yielding similar improvements in estimation error. In addition to reporting on the accuracy of Sun-BCNN and its impact on visual odometry, we analyze the sensitivity of our model to cloud cover, investigate the possibility of model transfer between urban and planetary analogue environments, and examine the impact of different methods for computing the mean and covariance of a norm-constrained vector on the accuracy and consistency of the estimated sun directions. Finally, we release Sun-BCNN as open-source software.},
author = {Valentin Peretroukhin and Lee Clement and Jonathan Kelly},
code = {https://github.com/utiasSTARS/sun-bcnn},
doi = {10.1177/0278364917749732},
journal = {The International Journal of Robotics Research},
month = {August},
number = {9},
pages = {996--1016},
title = {Inferring Sun Direction to Improve Visual Odometry: A Deep Learning Approach},
volume = {37},
year = {2018}
}
We present a method to incorporate global orientation information from the sun into a visual odometry pipeline using only the existing image stream, in which the sun is typically not visible. We leverage recent advances in Bayesian convolutional neural networks (BCNNs) to train and implement a sun detection model (dubbed Sun-BCNN) that infers a 3D sun direction vector from a single RGB image. Crucially, our method also computes a principled uncertainty associated with each prediction, using a Monte Carlo dropout scheme. We incorporate this uncertainty into a sliding window stereo visual odometry pipeline where accurate uncertainty estimates are critical for optimal data fusion. We evaluate our method on 21.6 km of urban driving data from the KITTI odometry benchmark where it achieves a median error of approximately 12 degrees and yields improvements of up to 42\% in translational average root mean squared error (ARMSE) and 32\% in rotational ARMSE compared with standard visual odometry. We further evaluate our method on an additional 10 km of visual navigation data from the Devon Island Rover Navigation dataset, achieving a median error of less than 8 degrees and yielding similar improvements in estimation error. In addition to reporting on the accuracy of Sun-BCNN and its impact on visual odometry, we analyze the sensitivity of our model to cloud cover, investigate the possibility of model transfer between urban and planetary analogue environments, and examine the impact of different methods for computing the mean and covariance of a norm-constrained vector on the accuracy and consistency of the estimated sun directions. Finally, we release Sun-BCNN as open-source software.
-
Near-Optimal Budgeted Data Exchange for Distributed Loop Closure Detection
Y. Tian, K. Khosoussi, M. Giamou, J. Kelly, and J. How
Proceedings of Robotics: Science and Systems (RSS’18), Pittsburgh, Pennsylvania, USA, Jun. 26–28, 2018. Bibtex |
Abstract |
arXiv |
Code @inproceedings{2018_Tian_Near-Optimal,
abstract = {Inter-robot loop closure detection is a core problem in collaborative SLAM (CSLAM). Establishing inter-robot loop closures is a resource-demanding process, during which robots must consume a substantial amount of mission-critical resources (e.g., battery and bandwidth) to exchange sensory data. However, even with the most resource-efficient techniques, the resources available onboard may be insufficient for verifying every potential loop closure. This work addresses this critical challenge by proposing a resource-adaptive framework for distributed loop closure detection. We seek to maximize task-oriented objectives subject to a budget constraint on total data transmission. This problem is in general NP-hard. We approach this problem from different perspectives and leverage existing results on monotone submodular maximization to provide efficient approximation algorithms with performance guarantees. The proposed approach is extensively evaluated using the KITTI odometry benchmark dataset and synthetic Manhattan-like datasets.},
address = {Pittsburgh, Pennsylvania, USA},
author = {Yulun Tian and Kasra Khosoussi and Matthew Giamou and Jonathan Kelly and Jonathan How},
booktitle = {Proceedings of Robotics: Science and Systems {(RSS'18)}},
code = {https://github.com/utiasSTARS/cslam-resource},
date = {2018-06-26/2018-06-28},
month = {Jun. 26--28},
title = {Near-Optimal Budgeted Data Exchange for Distributed Loop Closure Detection},
url = {http://www.roboticsproceedings.org/rss14/p71.pdf},
year = {2018}
}
Inter-robot loop closure detection is a core problem in collaborative SLAM (CSLAM). Establishing inter-robot loop closures is a resource-demanding process, during which robots must consume a substantial amount of mission-critical resources (e.g., battery and bandwidth) to exchange sensory data. However, even with the most resource-efficient techniques, the resources available onboard may be insufficient for verifying every potential loop closure. This work addresses this critical challenge by proposing a resource-adaptive framework for distributed loop closure detection. We seek to maximize task-oriented objectives subject to a budget constraint on total data transmission. This problem is in general NP-hard. We approach this problem from different perspectives and leverage existing results on monotone submodular maximization to provide efficient approximation algorithms with performance guarantees. The proposed approach is extensively evaluated using the KITTI odometry benchmark dataset and synthetic Manhattan-like datasets.
-
Load Sharing — Obstacle Avoidance and Admittance Control on a Mobile Manipulator
T. Ulrich
Master Thesis , Swiss Federal Institute of Technology Zurich, Zurich, Switzerland, 2018. Bibtex |
Abstract @mastersthesis{2018_Ulrich_Load,
abstract = {We present an implementation of a load-sharing algorithm between a human and a robot partner, designed to jointly carry an object in an indoor cluttered environment, of which the robot has no prior. We review the state of human-robot interaction in general and deploy cooperation, using information exchange through forces and torque applied to the jointly handled object. The work is set within the master-slave paradigm and combines an admittance controller with obstacle avoidance to ensure pro-active behaviour on the robot side and collision free trajectories at all times. We derive the implementation from existing literature and validate the working algorithm on a mobile manipulator, consisting of a Clearpath Ridgeback platform, a Universal Robot 10 and a Robotiq three finger gripper.},
address = {Zurich, Switzerland},
author = {Tobias Ulrich},
month = {September},
school = {Swiss Federal Institute of Technology Zurich},
title = {Load Sharing -- Obstacle Avoidance and Admittance Control on a Mobile Manipulator},
year = {2018}
}
We present an implementation of a load-sharing algorithm between a human and a robot partner, designed to jointly carry an object in an indoor cluttered environment, of which the robot has no prior. We review the state of human-robot interaction in general and deploy cooperation, using information exchange through forces and torque applied to the jointly handled object. The work is set within the master-slave paradigm and combines an admittance controller with obstacle avoidance to ensure pro-active behaviour on the robot side and collision free trajectories at all times. We derive the implementation from existing literature and validate the working algorithm on a mobile manipulator, consisting of a Clearpath Ridgeback platform, a Universal Robot 10 and a Robotiq three finger gripper.
-
LSTM-Based Zero-Velocity Detection for Robust Inertial Navigation
B. Wagstaff and J. Kelly
Proceedings of the International Conference on Indoor Positioning and Indoor Navigation (IPIN’18), Nantes, France, Sep. 24–27, 2018. DOI |
Bibtex |
Abstract |
arXiv |
Video @inproceedings{2018_Wagstaff_LSTM-Based,
abstract = {We present a method to improve the accuracy of a zero-velocity-aided inertial navigation system (INS) by replacing the standard zero-velocity detector with a long short-term memory (LSTM) neural network. While existing threshold-based zero-velocity detectors are not robust to varying motion types, our learned model accurately detects stationary periods of the inertial measurement unit (IMU) despite changes in the motion of the user. Upon detection, zero-velocity pseudo-measurements are fused with a dead reckoning motion model in an extended Kalman filter (EKF). We demonstrate that our LSTM-based zero-velocity detector, used within a zero-velocity-aided INS, improves zero-velocity detection during human localization tasks. Consequently, localization accuracy is also improved.
Our system is evaluated on more than 7.5 km of indoor pedestrian locomotion data, acquired from five different subjects. We show that 3D positioning error is reduced by over 34\% compared to existing fixed-threshold zero-velocity detectors for walking, running, and stair climbing motions. Additionally, we demonstrate how our learned zero-velocity detector operates effectively during crawling and ladder climbing. Our system is calibration-free (no careful threshold-tuning is required) and operates consistently with differing users, IMU placements, and shoe types, while being compatible with any generic zero-velocity-aided INS.},
address = {Nantes, France},
author = {Brandon Wagstaff and Jonathan Kelly},
booktitle = {Proceedings of the International Conference on Indoor Positioning and Indoor Navigation {(IPIN'18)}},
date = {2018-09-24/2018-09-27},
doi = {10.1109/IPIN.2018.8533770},
month = {Sep. 24--27},
note = {Best Student Paper Runner-Up},
title = {LSTM-Based Zero-Velocity Detection for Robust Inertial Navigation},
url = {http://arxiv.org/abs/1807.05275},
video1 = {https://www.youtube.com/watch?v=PhmZ8NMoh2s},
year = {2018}
}
We present a method to improve the accuracy of a zero-velocity-aided inertial navigation system (INS) by replacing the standard zero-velocity detector with a long short-term memory (LSTM) neural network. While existing threshold-based zero-velocity detectors are not robust to varying motion types, our learned model accurately detects stationary periods of the inertial measurement unit (IMU) despite changes in the motion of the user. Upon detection, zero-velocity pseudo-measurements are fused with a dead reckoning motion model in an extended Kalman filter (EKF). We demonstrate that our LSTM-based zero-velocity detector, used within a zero-velocity-aided INS, improves zero-velocity detection during human localization tasks. Consequently, localization accuracy is also improved. Our system is evaluated on more than 7.5 km of indoor pedestrian locomotion data, acquired from five different subjects. We show that 3D positioning error is reduced by over 34\% compared to existing fixed-threshold zero-velocity detectors for walking, running, and stair climbing motions. Additionally, we demonstrate how our learned zero-velocity detector operates effectively during crawling and ladder climbing. Our system is calibration-free (no careful threshold-tuning is required) and operates consistently with differing users, IMU placements, and shoe types, while being compatible with any generic zero-velocity-aided INS.
Best Student Paper Runner-Up
-
Cheap or Robust? The Practical Realization of Self-Driving Wheelchair Technology
M. Burhanpurkar, M. Labbe, X. Gong, C. Guan, F. Michaud, and J. Kelly
Proceedings of the IEEE International Conference on Rehabilitation Robotics (ICORR’17), London, United Kingdom, Jul. 17–20, 2017, pp. 1079-1086. DOI |
Bibtex |
Abstract |
arXiv @inproceedings{2017_Burhanpurkar_Cheap,
abstract = {To date, self-driving experimental wheelchair tech- nologies have been either inexpensive or robust, but not both. Yet, in order to achieve real-world acceptance, both qualities are fundamentally essential. We present a unique approach to achieve inexpensive and robust autonomous and semi-autonomous assistive navigation for existing fielded wheelchairs, of which there are approximately 5 million units in Canada and United States alone. Our prototype wheelchair platform is capable of localization and mapping, as well as robust obstacle avoidance, using only a commodity RGB-D sensor and wheel odometry. As a specific example of the navigation capabilities, we focus on the single most common navigation problem: the traversal of narrow doorways in arbitrary environments. The software we have developed is generalizable to corridor following, desk docking, and other navigation tasks that are either extremely difficult or impossible for people with upper-body mobility impairments.},
address = {London, United Kingdom},
author = {Maya Burhanpurkar and Mathieu Labbe and Xinyi Gong and Charlie Guan and Francois Michaud and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE} International Conference on Rehabilitation Robotics {(ICORR'17)}},
date = {2017-07-17/2017-07-20},
doi = {10.1109/ICORR.2017.8009393},
month = {Jul. 17--20},
pages = {1079--1086},
title = {Cheap or Robust? {The} Practical Realization of Self-Driving Wheelchair Technology},
url = {https://arxiv.org/abs/1707.05301},
year = {2017}
}
To date, self-driving experimental wheelchair tech- nologies have been either inexpensive or robust, but not both. Yet, in order to achieve real-world acceptance, both qualities are fundamentally essential. We present a unique approach to achieve inexpensive and robust autonomous and semi-autonomous assistive navigation for existing fielded wheelchairs, of which there are approximately 5 million units in Canada and United States alone. Our prototype wheelchair platform is capable of localization and mapping, as well as robust obstacle avoidance, using only a commodity RGB-D sensor and wheel odometry. As a specific example of the navigation capabilities, we focus on the single most common navigation problem: the traversal of narrow doorways in arbitrary environments. The software we have developed is generalizable to corridor following, desk docking, and other navigation tasks that are either extremely difficult or impossible for people with upper-body mobility impairments.
-
Improving the Accuracy of Stereo Visual Odometry Using Visual Illumination Estimation
L. Clement, V. Peretroukhin, and J. Kelly
in 2016 International Symposium on Experimental Robotics , D. Kulic, Y. Nakamura, O. Khatib, and G. Venture, Eds., Cham: Springer International Publishing AG, 2017, vol. 1, pp. 409-419. DOI |
Bibtex |
Abstract |
arXiv @incollection{2017_Clement_Improving,
abstract = {In the absence of reliable and accurate GPS, visual odometry (VO) has emerged as an effective means of estimating the egomotion of robotic vehicles. Like any dead-reckoning technique, VO suffers from unbounded accumulation of drift error over time, but this accumulation can be limited by incorporating absolute orientation information from, for example, a sun sensor. In this paper, we leverage recent work on visual outdoor illumination estimation to show that estimation error in a stereo VO pipeline can be reduced by inferring the sun position from the same image stream used to compute VO, thereby gaining the benefits of sun sensing without requiring a dedicated sun sensor or the sun to be visible to the camera. We compare sun estimation methods based on hand-crafted visual cues and Convolutional Neural Networks (CNNs) and demonstrate our approach on a combined 7.8 km of urban driving from the popular KITTI dataset, achieving up to a 43\% reduction in translational average root mean squared error (ARMSE) and a 59\% reduction in final translational drift error compared to pure VO alone.},
address = {Cham},
author = {Lee Clement and Valentin Peretroukhin and Jonathan Kelly},
booktitle = {2016 International Symposium on Experimental Robotics},
doi = {https://doi.org/10.1007/978-3-319-50115-4_36},
editor = {Dana Kulic and Yoshihiko Nakamura and Oussama Khatib and Gentiane Venture},
isbn = {978-3-319-50114-7},
note = {Invited to Journal Special Issue},
pages = {409--419},
publisher = {Springer International Publishing AG},
series = {Springer Proceedings in Advanced Robotics},
title = {Improving the Accuracy of Stereo Visual Odometry Using Visual Illumination Estimation},
url = {https://arxiv.org/abs/1609.04705},
volume = {1},
year = {2017}
}
In the absence of reliable and accurate GPS, visual odometry (VO) has emerged as an effective means of estimating the egomotion of robotic vehicles. Like any dead-reckoning technique, VO suffers from unbounded accumulation of drift error over time, but this accumulation can be limited by incorporating absolute orientation information from, for example, a sun sensor. In this paper, we leverage recent work on visual outdoor illumination estimation to show that estimation error in a stereo VO pipeline can be reduced by inferring the sun position from the same image stream used to compute VO, thereby gaining the benefits of sun sensing without requiring a dedicated sun sensor or the sun to be visible to the camera. We compare sun estimation methods based on hand-crafted visual cues and Convolutional Neural Networks (CNNs) and demonstrate our approach on a combined 7.8 km of urban driving from the popular KITTI dataset, achieving up to a 43\% reduction in translational average root mean squared error (ARMSE) and a 59\% reduction in final translational drift error compared to pure VO alone.
Invited to Journal Special Issue
-
Robust Monocular Visual Teach and Repeat Aided by Local Ground Planarity and Colour-Constant Imagery
L. Clement, J. Kelly, and T. D. Barfoot
Journal of Field Robotics, vol. 34, iss. 1, pp. 74-97, 2017. DOI |
Bibtex |
Abstract @article{2017_Clement_Robust,
abstract = {Visual Teach and Repeat (VT&R) allows an autonomous vehicle to accurately repeat a previously traversed route using only vision sensors. Most VT&R systems rely on natively 3D sensors such as stereo cameras for mapping and localization, but many existing mobile robots are equipped with only 2D monocular vision, typically for teleoperation. In this paper, we extend VT&R to the most basic sensor configuration -- a single monocular camera. We show that kilometer-scale route repetition can be achieved with centimeter-level accuracy by approximating the local ground surface near the vehicle as a plane with some uncertainty. This allows our system to recover absolute scale from the known position and orientation of the camera relative to the vehicle, which simplifies threshold-based outlier rejection and the estimation and control of lateral path-tracking error --- essential components of high-accuracy route repetition. We enhance the robustness of our monocular VT&R system to common failure cases through the use of color-constant imagery, which provides it with a degree of resistance to lighting changes and moving shadows where keypoint matching on standard grey images tends to struggle. Through extensive testing on a combined 30km of autonomous navigation data collected on multiple vehicles in a variety of highly non-planar terrestrial and planetary-analogue environments, we demonstrate that our system is capable of achieving route-repetition accuracy on par with its stereo counterpart, with only a modest trade-off in robustness.},
author = {Lee Clement and Jonathan Kelly and Timothy D. Barfoot},
doi = {10.1002/rob.21655},
journal = {Journal of Field Robotics},
month = {January},
number = {1},
pages = {74--97},
title = {Robust Monocular Visual Teach and Repeat Aided by Local Ground Planarity and Colour-Constant Imagery},
volume = {34},
year = {2017}
}
Visual Teach and Repeat (VT&R) allows an autonomous vehicle to accurately repeat a previously traversed route using only vision sensors. Most VT&R systems rely on natively 3D sensors such as stereo cameras for mapping and localization, but many existing mobile robots are equipped with only 2D monocular vision, typically for teleoperation. In this paper, we extend VT&R to the most basic sensor configuration -- a single monocular camera. We show that kilometer-scale route repetition can be achieved with centimeter-level accuracy by approximating the local ground surface near the vehicle as a plane with some uncertainty. This allows our system to recover absolute scale from the known position and orientation of the camera relative to the vehicle, which simplifies threshold-based outlier rejection and the estimation and control of lateral path-tracking error --- essential components of high-accuracy route repetition. We enhance the robustness of our monocular VT&R system to common failure cases through the use of color-constant imagery, which provides it with a degree of resistance to lighting changes and moving shadows where keypoint matching on standard grey images tends to struggle. Through extensive testing on a combined 30km of autonomous navigation data collected on multiple vehicles in a variety of highly non-planar terrestrial and planetary-analogue environments, we demonstrate that our system is capable of achieving route-repetition accuracy on par with its stereo counterpart, with only a modest trade-off in robustness.
-
Automatic and Featureless Sim(3) Calibration of Planar Lidars to Egomotion Sensors
J. Lambert
Master Thesis , University of Toronto, Toronto, Ontario, Canada, 2017. Bibtex |
Abstract @mastersthesis{2017_Lambert_Automatic,
abstract = {Autonomous and mobile robots often rely on the fusion of information from different sensors to accomplish important tasks. The prerequisite for successful data fusion is an accurate estimate of the coordinate transformation between the sensors. This thesis aims at generalizing the process of extrinsically calibrating two rigidly attached sensors on a mobile robot. An entropy-based, point cloud reconstruction technique is developed to calibrate a planar lidar to a sensor capable of providing egomotion information. Recent work in this area is revisited and its theory extended to the problem of recovering the Sim(3) transformation between a planar lidar and a monocular camera, where the scale of the camera trajectory is not known a priori. An efficient algorithm with only a single tuning parameter is implemented and studied. An experimental analysis of the algorithm demonstrates this parameter provides a trade-off between computational efficiency and cost function accuracy. The robustness of the approach is tested on realistic simula- tions in multiple environments, as well as on data collected from a hand-held sensor rig. Results show that, given a non-degenerate trajectory and a sufficient number of lidar measurements, the calibration procedure achieves millimetre-scale and sub-degree accuracy. Moreover, the method relaxes the need for specific scene geometry, fiducial markers, and overlapping sensor fields of view, which had previously limited similar techniques.},
address = {Toronto, Ontario, Canada},
author = {Jacob Lambert},
month = {January},
school = {University of Toronto},
title = {Automatic and Featureless Sim(3) Calibration of Planar Lidars to Egomotion Sensors},
year = {2017}
}
Autonomous and mobile robots often rely on the fusion of information from different sensors to accomplish important tasks. The prerequisite for successful data fusion is an accurate estimate of the coordinate transformation between the sensors. This thesis aims at generalizing the process of extrinsically calibrating two rigidly attached sensors on a mobile robot. An entropy-based, point cloud reconstruction technique is developed to calibrate a planar lidar to a sensor capable of providing egomotion information. Recent work in this area is revisited and its theory extended to the problem of recovering the Sim(3) transformation between a planar lidar and a monocular camera, where the scale of the camera trajectory is not known a priori. An efficient algorithm with only a single tuning parameter is implemented and studied. An experimental analysis of the algorithm demonstrates this parameter provides a trade-off between computational efficiency and cost function accuracy. The robustness of the approach is tested on realistic simula- tions in multiple environments, as well as on data collected from a hand-held sensor rig. Results show that, given a non-degenerate trajectory and a sufficient number of lidar measurements, the calibration procedure achieves millimetre-scale and sub-degree accuracy. Moreover, the method relaxes the need for specific scene geometry, fiducial markers, and overlapping sensor fields of view, which had previously limited similar techniques.
-
From Global to Local: Maintaining Accurate Mobile Manipulator State Estimates Over Long Trajectories
F. Maric
Master Thesis , University of Zagreb, Zagreb, Croatia, 2017. Bibtex |
Abstract |
PDF @mastersthesis{2017_Maric_Maintaining,
abstract = {In this thesis the problem of performing a long trajectory while maintaining an accurate state estimate is explored in the case of a mobile manipulator. The mobile manipulator used consists of a 6 degree-of-freedom manipulator and a omni-directional platform. State estimation is performed using a probabilistic framework, fusing multiple velocity and position estimates. Two approaches are explored for motion planning, the classical task priority approach and the more contemporary sequential convex optimization. Software implementation details are presented and tests are performed on both the simulation and real robot. The results show satisfactory trajectory following performance using local state estimates and motion planning.},
address = {Zagreb, Croatia},
author = {Filip Maric},
month = {September},
school = {University of Zagreb},
title = {From Global to Local: Maintaining Accurate Mobile Manipulator State Estimates Over Long Trajectories},
year = {2017}
}
In this thesis the problem of performing a long trajectory while maintaining an accurate state estimate is explored in the case of a mobile manipulator. The mobile manipulator used consists of a 6 degree-of-freedom manipulator and a omni-directional platform. State estimation is performed using a probabilistic framework, fusing multiple velocity and position estimates. Two approaches are explored for motion planning, the classical task priority approach and the more contemporary sequential convex optimization. Software implementation details are presented and tests are performed on both the simulation and real robot. The results show satisfactory trajectory following performance using local state estimates and motion planning.
-
Reducing Drift in Visual Odometry by Inferring Sun Direction Using a Bayesian Convolutional Neural Network
V. Peretroukhin, L. Clement, and J. Kelly
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’17), Singapore, May 29–Jun. 3, 2017, pp. 2035-2042. DOI |
Bibtex |
Abstract |
PDF |
arXiv |
Video @inproceedings{2017_Peretroukhin_Reducing,
abstract = {We present a method to incorporate global orientation information from the sun into a visual odometry pipeline using the existing image stream only. We leverage recent advances in Bayesian Convolutional Neural Networks to train and implement a sun detection model that infers a three-dimensional sun direction vector from a single RGB image (where the sun is typically not visible). Crucially, our method also computes a principled uncertainty associated with each prediction, using a Monte-Carlo dropout scheme. We incorporate this uncertainty into a sliding window stereo visual odometry pipeline where accurate uncertainty estimates are critical for optimal data fusion. Our Bayesian sun detection model achieves median errors of less than 10 degrees on the KITTI odometry benchmark training set, and yields improvements of up to 37\% in translational ARMSE and 32\% in rotational ARMSE compared to standard VO. An implementation of our Bayesian CNN sun estimator (Sun-BCNN) is available as open-source code at https://github.com/utiasSTARS/sun-bcnn-vo.},
address = {Singapore},
author = {Valentin Peretroukhin and Lee Clement and Jonathan Kelly},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'17})},
date = {2017-05-29/2017-06-03},
doi = {10.1109/ICRA.2017.7989235},
month = {May 29--Jun. 3},
pages = {2035--2042},
title = {Reducing Drift in Visual Odometry by Inferring Sun Direction Using a Bayesian Convolutional Neural Network},
url = {https://arxiv.org/abs/1609.05993},
video1 = {https://www.youtube.com/watch?v=c5XTrq3a2tE},
year = {2017}
}
We present a method to incorporate global orientation information from the sun into a visual odometry pipeline using the existing image stream only. We leverage recent advances in Bayesian Convolutional Neural Networks to train and implement a sun detection model that infers a three-dimensional sun direction vector from a single RGB image (where the sun is typically not visible). Crucially, our method also computes a principled uncertainty associated with each prediction, using a Monte-Carlo dropout scheme. We incorporate this uncertainty into a sliding window stereo visual odometry pipeline where accurate uncertainty estimates are critical for optimal data fusion. Our Bayesian sun detection model achieves median errors of less than 10 degrees on the KITTI odometry benchmark training set, and yields improvements of up to 37\% in translational ARMSE and 32\% in rotational ARMSE compared to standard VO. An implementation of our Bayesian CNN sun estimator (Sun-BCNN) is available as open-source code at https://github.com/utiasSTARS/sun-bcnn-vo.
-
Editorial: Special Issue on Field and Service Robotics
F. Pomerleau and J. Kelly
Journal of Field Robotics, vol. 34, iss. 1, pp. 3-4, 2017. DOI |
Bibtex |
PDF @article{2017_Pomerleau_FSR,
author = {Francois Pomerleau and Jonathan Kelly},
doi = {10.1002/rob.21703},
journal = {Journal of Field Robotics},
month = {January},
number = {1},
pages = {3--4},
title = {Editorial: Special Issue on Field and Service Robotics},
volume = {34},
year = {2017}
}
-
Increasing Persistent Navigation Capabilities for Underwater Vehicles with Augmented Terrain-Based Navigation
G. M. Reis, M. Fitzpatrick, J. Anderson, J. Kelly, L. Bobadilla, and R. N. Smith
Proceedings of the MTS/IEEE Oceans Conference (OCEANS’17), Aberdeen, United Kingdom, Jun. 19–22, 2017. DOI |
Bibtex |
Abstract |
PDF @inproceedings{2017_Reis_Increasing,
abstract = {Accurate and energy-efficient navigation and localization methods for autonomous underwater vehicles continues to be an active area of research. As interesting as they are important, ocean processes are spatiotemporally dynamic and their study requires vehicles that can maneuver and sample intelligently while underwater for extended durations. In this paper, we present a new technique for augmenting terrain-based navigation with physical water data to enhance the utility of traditional methods for navigation and localization. We examine the construct of this augmentation method over a range of deployment regions, e.g., ocean and freshwater lake. Data from field trials are presented and analyzed for multiple deployments of an autonomous underwater vehicle.},
address = {Aberdeen, United Kingdom},
author = {Gregory Murad Reis and Michael Fitzpatrick and Jacob Anderson and Jonathan Kelly and Leonardo Bobadilla and Ryan N. Smith},
booktitle = {Proceedings of the {MTS/IEEE} Oceans Conference {(OCEANS'17)}},
date = {2017-06-19/2017-06-22},
doi = {10.1109/OCEANSE.2017.8084815},
month = {Jun. 19--22},
note = {Best Student Paper Finalist},
title = {Increasing Persistent Navigation Capabilities for Underwater Vehicles with Augmented Terrain-Based Navigation},
year = {2017}
}
Accurate and energy-efficient navigation and localization methods for autonomous underwater vehicles continues to be an active area of research. As interesting as they are important, ocean processes are spatiotemporally dynamic and their study requires vehicles that can maneuver and sample intelligently while underwater for extended durations. In this paper, we present a new technique for augmenting terrain-based navigation with physical water data to enhance the utility of traditional methods for navigation and localization. We examine the construct of this augmentation method over a range of deployment regions, e.g., ocean and freshwater lake. Data from field trials are presented and analyzed for multiple deployments of an autonomous underwater vehicle.
Best Student Paper Finalist
-
Improving Foot-Mounted Inertial Navigation Through Real-Time Motion Classification
B. Wagstaff, V. Peretroukhin, and J. Kelly
Proceedings of the International Conference on Indoor Positioning and Indoor Navigation (IPIN’17), Sapporo, Japan, Sep. 18–21, 2017. DOI |
Bibtex |
Abstract |
arXiv |
Video @inproceedings{2017_Wagstaff_Improving,
abstract = {We present a method to improve the accuracy of a foot-mounted, zero-velocity-aided inertial navigation system (INS) by varying estimator parameters based on a real-time classification of motion type. We train a support vector machine (SVM) classifier using inertial data recorded by a single foot-mounted sensor to differentiate between six motion types (walking, jogging, running, sprinting, crouch-walking, and ladder-climbing) and report mean test classification accuracy of over 90\% on a dataset with five different subjects.
From these motion types, we select two of the most common (walking and running), and describe a method to compute optimal zero-velocity detection parameters tailored to both a specific user and motion type by maximizing the detector F-score. By combining the motion classifier with a set of optimal detection parameters, we show how we can reduce INS position error during mixed walking and running motion. We evaluate our adaptive system on a total of 5.9 km of indoor pedestrian navigation performed by five different subjects moving along a 130 m path with surveyed ground truth markers.},
address = {Sapporo, Japan},
author = {Brandon Wagstaff and Valentin Peretroukhin and Jonathan Kelly},
booktitle = {Proceedings of the International Conference on Indoor Positioning and Indoor Navigation {(IPIN'17)}},
date = {2017-09-18/2017-09-21},
doi = {10.1109/IPIN.2017.8115947},
month = {Sep. 18--21},
title = {Improving Foot-Mounted Inertial Navigation Through Real-Time Motion Classification},
url = {http://arxiv.org/abs/1707.01152},
video1 = {https://www.youtube.com/watch?v=Jiqj6j9E8dI},
year = {2017}
}
We present a method to improve the accuracy of a foot-mounted, zero-velocity-aided inertial navigation system (INS) by varying estimator parameters based on a real-time classification of motion type. We train a support vector machine (SVM) classifier using inertial data recorded by a single foot-mounted sensor to differentiate between six motion types (walking, jogging, running, sprinting, crouch-walking, and ladder-climbing) and report mean test classification accuracy of over 90\% on a dataset with five different subjects. From these motion types, we select two of the most common (walking and running), and describe a method to compute optimal zero-velocity detection parameters tailored to both a specific user and motion type by maximizing the detector F-score. By combining the motion classifier with a set of optimal detection parameters, we show how we can reduce INS position error during mixed walking and running motion. We evaluate our adaptive system on a total of 5.9 km of indoor pedestrian navigation performed by five different subjects moving along a 130 m path with surveyed ground truth markers.
-
Determining the Time Delay Between Inertial and Visual Sensor Measurements
J. Kelly, N. Roy, and G. S. Sukhatme
IEEE Transactions on Robotics, vol. 30, iss. 6, pp. 1514-1523, 2014. DOI |
Bibtex |
Abstract @article{2014_Kelly_Determining,
abstract = {We examine the problem of determining the relative time delay between IMU and camera data streams. The primary difficulty is that the correspondences between measurements from the sensors are not initially known, and hence, the time delay cannot be computed directly. We instead formulate time delay calibration as a registration problem, and introduce a calibration algorithm that operates by aligning curves in a three-dimensional orientation space. Results from simulation studies and from experiments with real hardware demonstrate that the delay can be accurately calibrated.},
author = {Jonathan Kelly and Nicholas Roy and Gaurav S. Sukhatme},
doi = {10.1109/TRO.2014.2343073},
journal = {{IEEE} Transactions on Robotics},
month = {December},
number = {6},
pages = {1514--1523},
title = {Determining the Time Delay Between Inertial and Visual Sensor Measurements},
volume = {30},
year = {2014}
}
We examine the problem of determining the relative time delay between IMU and camera data streams. The primary difficulty is that the correspondences between measurements from the sensors are not initially known, and hence, the time delay cannot be computed directly. We instead formulate time delay calibration as a registration problem, and introduce a calibration algorithm that operates by aligning curves in a three-dimensional orientation space. Results from simulation studies and from experiments with real hardware demonstrate that the delay can be accurately calibrated.
-
A General Framework for Temporal Calibration of Multiple Proprioceptive and Exteroceptive Sensors
J. Kelly and G. S. Sukhatme
in Experimental Robotics: The 12th International Symposium on Experimental Robotics , O. Khatib, V. Kumar, and G. S. Sukhatme, Eds., Berlin, Heidelberg: Springer, 2014, vol. 79, pp. 195-209. DOI |
Bibtex |
Abstract |
PDF @incollection{2014_Kelly_General,
abstract = {Fusion of data from multiple sensors can enable robust navigation in varied environments. However, for optimal performance, the sensors must be calibrated relative to one another. Full sensor-to-sensor calibration is a spatiotemporal problem: we require an accurate estimate of the relative timing of measurements for each pair of sensors, in addition to the 6-DOF sensor-to-sensor transform. In this paper, we examine the problem of determining the time delays between multiple proprioceptive and exteroceptive sensor data streams. The primary difficultly is that the correspondences between measurements from different sensors are unknown, and hence the delays cannot be computed directly. We instead formulate temporal calibration as a registration task. Our algorithm operates by aligning curves in a three-dimensional orientation space, and, as such, can be considered as a variant of Iterative Closest Point (ICP). We present results from simulation studies and from experiments with a PR2 robot, which demonstrate accurate calibration of the time delays between measurements from multiple, heterogeneous sensors.},
address = {Berlin, Heidelberg},
author = {Jonathan Kelly and Gaurav S. Sukhatme},
booktitle = {Experimental Robotics: The 12th International Symposium on Experimental Robotics},
doi = {10.1007/978-3-642-28572-1_14},
editor = {Oussama Khatib and Vijay Kumar and Gaurav S. Sukhatme},
isbn = {978-3-642-28571-4},
pages = {195--209},
publisher = {Springer},
series = {Springer Tracts in Advanced Robotics},
title = {A General Framework for Temporal Calibration of Multiple Proprioceptive and Exteroceptive Sensors},
volume = {79},
year = {2014}
}
Fusion of data from multiple sensors can enable robust navigation in varied environments. However, for optimal performance, the sensors must be calibrated relative to one another. Full sensor-to-sensor calibration is a spatiotemporal problem: we require an accurate estimate of the relative timing of measurements for each pair of sensors, in addition to the 6-DOF sensor-to-sensor transform. In this paper, we examine the problem of determining the time delays between multiple proprioceptive and exteroceptive sensor data streams. The primary difficultly is that the correspondences between measurements from different sensors are unknown, and hence the delays cannot be computed directly. We instead formulate temporal calibration as a registration task. Our algorithm operates by aligning curves in a three-dimensional orientation space, and, as such, can be considered as a variant of Iterative Closest Point (ICP). We present results from simulation studies and from experiments with a PR2 robot, which demonstrate accurate calibration of the time delays between measurements from multiple, heterogeneous sensors.
-
Optimizing Camera Perspective for Stereo Visual Odometry
V. Peretroukhin, J. Kelly, and T. D. Barfoot
Proceedings of the Canadian Conference on Computer and Robot Vision (CRV’14), Montreal, Quebec, Canada, May 7–9, 2014, pp. 1-7. DOI |
Bibtex |
Abstract |
PDF @inproceedings{2014_Peretroukhin_Optimizing,
abstract = {Visual Odometry (VO) is an integral part of many navigation techniques in mobile robotics. In this work, we investigate how the orientation of the camera affects the overall position estimates recovered from stereo VO. Through simulations and experimental work, we demonstrate that this error can be significantly reduced by changing the perspective of the stereo camera in relation to the moving platform. Specifically, we show that orienting the camera at an oblique angle to the direction of travel can reduce VO error by up to 82\% in simulations and up to 59\% in experimental data. A variety of parameters are investigated for their effects on this trend including frequency of captured images and camera resolution.},
address = {Montreal, Quebec, Canada},
author = {Valentin Peretroukhin and Jonathan Kelly and Timothy D. Barfoot},
booktitle = {Proceedings of the Canadian Conference on Computer and Robot Vision {(CRV'14)}},
date = {2014-05-07/2014-05-09},
doi = {10.1109/CRV.2014.9},
month = {May 7--9},
pages = {1--7},
title = {Optimizing Camera Perspective for Stereo Visual Odometry},
year = {2014}
}
Visual Odometry (VO) is an integral part of many navigation techniques in mobile robotics. In this work, we investigate how the orientation of the camera affects the overall position estimates recovered from stereo VO. Through simulations and experimental work, we demonstrate that this error can be significantly reduced by changing the perspective of the stereo camera in relation to the moving platform. Specifically, we show that orienting the camera at an oblique angle to the direction of travel can reduce VO error by up to 82\% in simulations and up to 59\% in experimental data. A variety of parameters are investigated for their effects on this trend including frequency of captured images and camera resolution.
-
An autonomous manipulation system based on force control and optimization
L. Righetti, M. Kalakrishnan, P. Pastor, J. Binney, J. Kelly, R. C. Voorhies, G. S. Sukhatme, and S. Schaal
Autonomous Robots, vol. 36, iss. 1–2, pp. 11-30, 2014. DOI |
Bibtex |
Abstract @article{2014_Righetti_Autonomous,
abstract = {In this paper we present an architecture for autonomous manipulation. Our approach is based on the belief that contact interactions during manipulation should be exploited to improve dexterity and that optimizing motion plans is useful to create more robust and repeatable manipulation behaviors. We therefore propose an architecture where state of the art force/torque control and optimization-based motion planning are the core components of the system. We give a detailed description of the modules that constitute the complete system and discuss the challenges inherent to creat- ing such a system. We present experimental results for several grasping and manipulation tasks to demonstrate the performance and robustness of our approach.},
author = {Ludovic Righetti and Mrinal Kalakrishnan and Peter Pastor and Jonathan Binney and Jonathan Kelly and Randolph C. Voorhies and Gaurav S. Sukhatme and Stefan Schaal},
doi = {10.1007/s10514-013-9365-9},
journal = {Autonomous Robots},
month = {January},
number = {1--2},
pages = {11--30},
title = {An autonomous manipulation system based on force control and optimization},
volume = {36},
year = {2014}
}
In this paper we present an architecture for autonomous manipulation. Our approach is based on the belief that contact interactions during manipulation should be exploited to improve dexterity and that optimizing motion plans is useful to create more robust and repeatable manipulation behaviors. We therefore propose an architecture where state of the art force/torque control and optimization-based motion planning are the core components of the system. We give a detailed description of the modules that constitute the complete system and discuss the challenges inherent to creat- ing such a system. We present experimental results for several grasping and manipulation tasks to demonstrate the performance and robustness of our approach.
-
Editorial: Special Issue on Long-Term Autonomy
T. Barfoot, J. Kelly, and G. Sibley
The International Journal of Robotics Research, vol. 32, iss. 14, pp. 1609-1610, 2013. DOI |
Bibtex |
PDF @article{2013_Barfoot_Long-Term,
author = {Tim Barfoot and Jonathan Kelly and Gabe Sibley},
doi = {10.1177/0278364913511182},
journal = {The International Journal of Robotics Research},
month = {December},
number = {14},
pages = {1609--1610},
title = {Editorial: Special Issue on Long-Term Autonomy},
volume = {32},
year = {2013}
}
-
Learning Task Error Models for Manipulation
P. Pastor, M. Kalakrishnan, J. Binney, J. Kelly, L. Righetti, G. S. Sukhatme, and S. Schaal
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’13), Karlsruhe, Germany, May 6–10, 2013, pp. 2612-2618. DOI |
Bibtex |
Abstract @inproceedings{2013_Pastor_Learning,
abstract = {Precise kinematic forward models are important for robots to successfully perform dexterous grasping and manipulation tasks, especially when visual servoing is rendered infeasible due to occlusions. A lot of research has been conducted to estimate geometric and non-geometric parameters of kinematic chains to minimize reconstruction errors. However, kinematic chains can include non-linearities, e.g. due to cable stretch and motor-side encoders, that result in significantly different errors for different parts of the state space. Previous work either does not consider such non-linearities or proposes to estimate non-geometric parameters of carefully engineered models that are robot specific. We propose a data-driven approach that learns task error models that account for such unmodeled non-linearities. We argue that in the context of grasping and manipulation, it is sufficient to achieve high accuracy in the task relevant state space. We identify this relevant state space using previously executed joint configurations and learn error corrections for those. Therefore, our system is developed to generate subsequent executions that are similar to previous ones. The experiments show that our method successfully captures the non-linearities in the head kinematic chain (due to a counter-balancing spring) and the arm kinematic chains (due to cable stretch) of the considered experimental platform, see Fig. 1. The feasibility of the presented error learning approach has also been evaluated in independent DARPA ARM-S testing contributing to successfully complete 67 out of 72 grasping and manipulation tasks.},
address = {Karlsruhe, Germany},
author = {Peter Pastor and Mrinal Kalakrishnan and Jonathan Binney and Jonathan Kelly and Ludovic Righetti and Gaurav S. Sukhatme and Stefan Schaal},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'13)}},
date = {2013-05-06/2013-05-10},
doi = {10.1109/ICRA.2013.6630935},
month = {May 6--10},
pages = {2612--2618},
title = {Learning Task Error Models for Manipulation},
year = {2013}
}
Precise kinematic forward models are important for robots to successfully perform dexterous grasping and manipulation tasks, especially when visual servoing is rendered infeasible due to occlusions. A lot of research has been conducted to estimate geometric and non-geometric parameters of kinematic chains to minimize reconstruction errors. However, kinematic chains can include non-linearities, e.g. due to cable stretch and motor-side encoders, that result in significantly different errors for different parts of the state space. Previous work either does not consider such non-linearities or proposes to estimate non-geometric parameters of carefully engineered models that are robot specific. We propose a data-driven approach that learns task error models that account for such unmodeled non-linearities. We argue that in the context of grasping and manipulation, it is sufficient to achieve high accuracy in the task relevant state space. We identify this relevant state space using previously executed joint configurations and learn error corrections for those. Therefore, our system is developed to generate subsequent executions that are similar to previous ones. The experiments show that our method successfully captures the non-linearities in the head kinematic chain (due to a counter-balancing spring) and the arm kinematic chains (due to cable stretch) of the considered experimental platform, see Fig. 1. The feasibility of the presented error learning approach has also been evaluated in independent DARPA ARM-S testing contributing to successfully complete 67 out of 72 grasping and manipulation tasks.
-
An Investigation on the Accuracy of Regional Ocean Models Through Field Trials
R. N. Smith, J. Kelly, K. Nazarzadeh, and G. S. Sukhatme
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’13), Karlsruhe, Germany, May 6–10, 2013, pp. 3436-3442. DOI |
Bibtex |
Abstract @inproceedings{2013_Smith_Investigation,
abstract = {Recent efforts in mission planning for underwater vehicles have utilised predictive models to aid in navigation, optimal path planning and drive opportunistic sampling. Although these models provide information at a unprecedented resolutions and have proven to increase accuracy and effectiveness in multiple campaigns, most are deterministic in nature. Thus, predictions cannot be incorporated into probabilistic planning frameworks, nor do they provide any metric on the variance or confidence of the output variables. In this paper, we provide an initial investigation into determining the confidence of ocean model predictions based on the results of multiple field deployments of two autonomous underwater vehicles.
For multiple missions of two autonomous gliders conducted over a two-month period in 2011, we compare actual vehicle executions to simulations of the same missions through the Regional Ocean Modeling System in an ocean region off the coast of southern California. This comparison provides a qualitative analysis of the current velocity predictions for areas within the selected deployment region. Ultimately, we present a spatial heat-map of the correlation between the ocean model predictions and the actual mission executions. Knowing where the model provides unreliable predictions can be incorporated into planners to increase the utility and application of the deterministic estimations.},
address = {Karlsruhe, Germany},
author = {Ryan N. Smith and Jonathan Kelly and Kimia Nazarzadeh and Gaurav S. Sukhatme},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'13)}},
date = {2013-05-06/2013-05-10},
doi = {10.1109/ICRA.2013.6631057},
month = {May 6--10},
pages = {3436--3442},
title = {An Investigation on the Accuracy of Regional Ocean Models Through Field Trials},
year = {2013}
}
Recent efforts in mission planning for underwater vehicles have utilised predictive models to aid in navigation, optimal path planning and drive opportunistic sampling. Although these models provide information at a unprecedented resolutions and have proven to increase accuracy and effectiveness in multiple campaigns, most are deterministic in nature. Thus, predictions cannot be incorporated into probabilistic planning frameworks, nor do they provide any metric on the variance or confidence of the output variables. In this paper, we provide an initial investigation into determining the confidence of ocean model predictions based on the results of multiple field deployments of two autonomous underwater vehicles. For multiple missions of two autonomous gliders conducted over a two-month period in 2011, we compare actual vehicle executions to simulations of the same missions through the Regional Ocean Modeling System in an ocean region off the coast of southern California. This comparison provides a qualitative analysis of the current velocity predictions for areas within the selected deployment region. Ultimately, we present a spatial heat-map of the correlation between the ocean model predictions and the actual mission executions. Knowing where the model provides unreliable predictions can be incorporated into planners to increase the utility and application of the deterministic estimations.
-
CELLO: A Fast Algorithm for Covariance Estimation
W. Vega-Brown, A. Bachrach, A. Bry, J. Kelly, and N. Roy
Proceedings of the IEEE International Conference on Robotics and Automation (ICRA’13), Karlsruhe, Germany, May 6–10, 2013, pp. 3160-3167. DOI |
Bibtex |
Abstract @inproceedings{2013_Vega-Brown_CELLO,
abstract = {We present CELLO (Covariance Estimation and Learning through Likelihood Optimization), an algorithm for predicting the covariances of measurements based on any available informative features. This algorithm is intended to improve the accuracy and reliability of on-line state estimation by providing a principled way to extend the conventional fixed-covariance Gaussian measurement model. We show that in ex- periments, CELLO learns to predict measurement covariances that agree with empirical covariances obtained by manually annotating sensor regimes. We also show that using the learned covariances during filtering provides substantial quantitative improvement to the overall state estimate.},
address = {Karlsruhe, Germany},
author = {William Vega-Brown and Abraham Bachrach and Adam Bry and Jonathan Kelly and Nicholas Roy},
booktitle = {Proceedings of the {IEEE} International Conference on Robotics and Automation {(ICRA'13)}},
date = {2013-05-06/2013-05-10},
doi = {10.1109/ICRA.2013.6631017},
month = {May 6--10},
pages = {3160--3167},
title = {{CELLO}: A Fast Algorithm for Covariance Estimation},
year = {2013}
}
We present CELLO (Covariance Estimation and Learning through Likelihood Optimization), an algorithm for predicting the covariances of measurements based on any available informative features. This algorithm is intended to improve the accuracy and reliability of on-line state estimation by providing a principled way to extend the conventional fixed-covariance Gaussian measurement model. We show that in ex- periments, CELLO learns to predict measurement covariances that agree with empirical covariances obtained by manually annotating sensor regimes. We also show that using the learned covariances during filtering provides substantial quantitative improvement to the overall state estimate.
-
Algorithmic Distributed Assembly
J. Kelly
Master Thesis , University of Alberta, Edmonton, Alberta, Canada, 2008. Bibtex |
Abstract @mastersthesis{2008_Kelly_Algorithmic,
abstract = {This thesis describes a model for planar distributed assembly, in which unit-square assembly components move randomly and independently on a two-dimensional grid, binding together to form a desired target structure. The components are simple reactive agents, with limited capabilities including short-range sensing and rule-based control only, and operate in an entirely decentralized manner.
Using the model, we investigate two primary issues, coordination and sensing, from an algorithmic perspective. Our goal is to determine how a group of components can be reliably programmed to produce a global result (structure) from purely local interactions. Towards this end, we define the local spatiotemporal ordering constraints that must be satisfied for assembly to progress in a coordinated manner, and give a procedure for encoding these constraints in a rule set. When executed by the components, this rule set is guaranteed to produce the target structure, despite the random actions of group members. We then introduce an optimization algorithm which is able to significantly reduce the number of distinct environmental states that components must recognize in order to assemble into a structure. Experiments show that our optimization algorithm outperforms existing approaches.},
address = {Edmonton, Alberta, Canada},
author = {Jonathan Kelly},
institution = {University of Alberta},
month = {April},
school = {University of Alberta},
title = {Algorithmic Distributed Assembly},
year = {2008}
}
This thesis describes a model for planar distributed assembly, in which unit-square assembly components move randomly and independently on a two-dimensional grid, binding together to form a desired target structure. The components are simple reactive agents, with limited capabilities including short-range sensing and rule-based control only, and operate in an entirely decentralized manner. Using the model, we investigate two primary issues, coordination and sensing, from an algorithmic perspective. Our goal is to determine how a group of components can be reliably programmed to produce a global result (structure) from purely local interactions. Towards this end, we define the local spatiotemporal ordering constraints that must be satisfied for assembly to progress in a coordinated manner, and give a procedure for encoding these constraints in a rule set. When executed by the components, this rule set is guaranteed to produce the target structure, despite the random actions of group members. We then introduce an optimization algorithm which is able to significantly reduce the number of distinct environmental states that components must recognize in order to assemble into a structure. Experiments show that our optimization algorithm outperforms existing approaches.
-
Combined Visual and Inertial Navigation for an Unmanned Aerial Vehicle
J. Kelly, S. Saripalli, and G. S. Sukhatme
in Field and Service Robotics: Results of the 6th International Conference , C. Laugier and R. Siegwart, Eds., Berlin Heidelberg: Springer, 2008, vol. 42/2008, pp. 255-264. DOI |
Bibtex |
Abstract |
PDF @incollection{2008_Kelly_Combined,
abstract = {We describe an UAV navigation system which combines stereo visual odometry with inertial measurements from an IMU. Our approach fuses the motion estimates from both sensors in an extended Kalman filter to determine vehicle position and attitude. We present results using data from a robotic helicopter, in which the visual and inertial system produced a final position estimate within 1\% of the measured GPS position, over a flight distance of more than 400 meters. Our results show that the combination of visual and inertial sensing reduced overall positioning error by nearly an order of magnitude compared to visual odometry alone.},
address = {Berlin Heidelberg},
author = {Jonathan Kelly and Srikanth Saripalli and Gaurav S. Sukhatme},
booktitle = {Field and Service Robotics: Results of the 6th International Conference},
doi = {10.1007/978-3-540-75404-6_24},
editor = {Christian Laugier and Roland Siegwart},
pages = {255--264},
publisher = {Springer},
series = {Springer Tracts in Advanced Robotics},
title = {Combined Visual and Inertial Navigation for an Unmanned Aerial Vehicle},
volume = {42/2008},
year = {2008}
}
We describe an UAV navigation system which combines stereo visual odometry with inertial measurements from an IMU. Our approach fuses the motion estimates from both sensors in an extended Kalman filter to determine vehicle position and attitude. We present results using data from a robotic helicopter, in which the visual and inertial system produced a final position estimate within 1\% of the measured GPS position, over a flight distance of more than 400 meters. Our results show that the combination of visual and inertial sensing reduced overall positioning error by nearly an order of magnitude compared to visual odometry alone.
-
On the Observability and Self-Calibration of Visual-Inertial Navigation Systems
J. Kelly
Los Angeles, California, USA, Tech. Rep. CRES-08-005, November, 2008. Bibtex |
Abstract @techreport{2008_Kelly_Observability,
abstract = {We examine the observability properties of visual-inertial navigation systems, with an emphasis on self-calibration of the six degrees-of-freedom rigid body transform between a camera and an inertial measurement unit (IMU). Our analysis depends on a differential geometric formulation of the calibration problem, and on an algebraic test for the observability rank condition, originally defined by Hermann and Krener.
We demonstrate that self-calibration of the camera-IMU transform is possible, under a variety of conditions. In contrast with previous work, we show that, in the presence of a known calibration target, both the local gravity vector and the IMU gyroscope and accelerometer biases are simultaneously observable (given sufficient excitation of the system). Further and more generally, we show that for a moving monocular camera and IMU, the absolute scene scale, gravity vector, and the IMU biases are all simultaneously observable. This result implies that full self-calibration is possible, without the need for any prior knowledge about the environment in which the system is operating.},
address = {Los Angeles, California, USA},
author = {Jonathan Kelly},
institution = {University of Southern California},
month = {November},
number = {CRES-08-005},
title = {On the Observability and Self-Calibration of Visual-Inertial Navigation Systems},
year = {2008}
}
We examine the observability properties of visual-inertial navigation systems, with an emphasis on self-calibration of the six degrees-of-freedom rigid body transform between a camera and an inertial measurement unit (IMU). Our analysis depends on a differential geometric formulation of the calibration problem, and on an algebraic test for the observability rank condition, originally defined by Hermann and Krener. We demonstrate that self-calibration of the camera-IMU transform is possible, under a variety of conditions. In contrast with previous work, we show that, in the presence of a known calibration target, both the local gravity vector and the IMU gyroscope and accelerometer biases are simultaneously observable (given sufficient excitation of the system). Further and more generally, we show that for a moving monocular camera and IMU, the absolute scene scale, gravity vector, and the IMU biases are all simultaneously observable. This result implies that full self-calibration is possible, without the need for any prior knowledge about the environment in which the system is operating.
-
A Note on Unscented Filtering and State Propagation in Nonlinear Systems
J. Kelly
Los Angeles, California, USA, Tech. Rep. CRES-08-004, November, 2008. Bibtex |
Abstract @techreport{2008_Kelly_Unscented,
abstract = {In this note, we illustrate the effect of nonlinear state propagation in the unscented Kalman filter (UKF). We consider a simple nonlinear system, consisting of a two-axis inertial measurement unit. Our intent is to show that the propagation of a set of sigma points through a nonlinear process model in the UKF can produce a counterintuitive (but correct) updated state estimate. We compare the results from the UKF with those from the well-known extended Kalman filter (EKF), to highlight how the UKF and the EKF differ.},
address = {Los Angeles, California, USA},
author = {Jonathan Kelly},
institution = {University of Southern California},
month = {November},
number = {CRES-08-004},
title = {A Note on Unscented Filtering and State Propagation in Nonlinear Systems},
year = {2008}
}
In this note, we illustrate the effect of nonlinear state propagation in the unscented Kalman filter (UKF). We consider a simple nonlinear system, consisting of a two-axis inertial measurement unit. Our intent is to show that the propagation of a set of sigma points through a nonlinear process model in the UKF can produce a counterintuitive (but correct) updated state estimate. We compare the results from the UKF with those from the well-known extended Kalman filter (EKF), to highlight how the UKF and the EKF differ.
-
Just Add Wheels: Leveraging Commodity Laptop Hardware for Robotics and AI Education
J. Kelly, J. Binney, A. Pereira, O. Khan, and G. S. Sukhatme
in AAAI Technical Report WS-08-02: Proceedings of the AAAI 2008 AI Education Colloquium , Z. Dodds, H. Hirsh, and K. Wagstaff, Eds., Menlo Park, California, USA: AAAI Press, 2008, pp. 50-55. Bibtex |
Abstract |
PDF @incollection{2008_Kelly_Wheels,
abstract = {Along with steady gains in processing power, commodity laptops are increasingly becoming sensor-rich devices. This trend, driven by consumer demand and enabled by improvements in solid-state sensor technology, offers an ideal opportunity to integrate robotics into K--12 and undergraduate education. By adding wheels, motors and a motor control board, a modern laptop can be transformed into a capable robot platform, for relatively little additional cost. We propose designing software and curricula around such platforms, leveraging hardware that many students already have in hand.
In this paper, we motivate our laptop-centric approach, and demonstrate a proof-of-concept laptop robot based on an Apple MacBook laptop and an iRobot Create mobile base. The MacBook is equipped with a built-in camera and a three-axis accelerometer unit -- we use the camera for monocular simultaneous localization and mapping (SLAM), and the accelerometer for 360 degree collision detection. The paper closes with some suggestions for ways in which to foster more work in this direction.},
address = {Menlo Park, California, USA},
author = {Jonathan Kelly and Jonathan Binney and Arvind Pereira and Omair Khan and Gaurav S. Sukhatme},
booktitle = {{AAAI} Technical Report WS-08-02: Proceedings of the {AAAI} 2008 AI Education Colloquium},
date = {2008-07-13},
editor = {Zachary Dodds and Haym Hirsh and Kiri Wagstaff},
isbn = {978-1-57735-370-6},
month = {Jul. 13},
pages = {50--55},
publisher = {AAAI Press},
title = {Just Add Wheels: Leveraging Commodity Laptop Hardware for Robotics and {AI} Education},
year = {2008}
}
Along with steady gains in processing power, commodity laptops are increasingly becoming sensor-rich devices. This trend, driven by consumer demand and enabled by improvements in solid-state sensor technology, offers an ideal opportunity to integrate robotics into K--12 and undergraduate education. By adding wheels, motors and a motor control board, a modern laptop can be transformed into a capable robot platform, for relatively little additional cost. We propose designing software and curricula around such platforms, leveraging hardware that many students already have in hand. In this paper, we motivate our laptop-centric approach, and demonstrate a proof-of-concept laptop robot based on an Apple MacBook laptop and an iRobot Create mobile base. The MacBook is equipped with a built-in camera and a three-axis accelerometer unit -- we use the camera for monocular simultaneous localization and mapping (SLAM), and the accelerometer for 360 degree collision detection. The paper closes with some suggestions for ways in which to foster more work in this direction.
