@article {919, title = {Environment Classification for Robotic Leg Prostheses and Exoskeletons using Deep Convolutional Neural Networks}, journal = {Frontiers in Neurorobotics}, volume = {15}, year = {2022}, month = {02/2022}, publisher = {bioRxiv}, abstract = {

Robotic leg prostheses and exoskeletons can provide powered locomotor assistance to older adults and/or persons with physical disabilities. However, the current locomotion mode recognition systems being developed for automated high-level control and decision-making rely on mechanical, inertial, and/or neuromuscular sensors, which inherently have limited prediction horizons (i.e., analogous to walking blindfolded). Inspired by the human vision-locomotor control system, we developed an environment classification system powered by computer vision and deep learning to predict the oncoming walking environments prior to physical interaction, therein allowing for more accurate and robust high-level control decisions. In this study, we first reviewed the development of our \“ExoNet\” database\—the largest and most diverse open-source dataset of wearable camera images of indoor and outdoor real-world walking environments, which were annotated using a hierarchical labeling architecture. We then trained and tested over a dozen state-of-the-art deep convolutional neural networks (CNNs) on the ExoNet database for image classification and automatic feature engineering, including: EfficientNetB0, InceptionV3, MobileNet, MobileNetV2, VGG16, VGG19, Xception, ResNet50, ResNet101, ResNet152, DenseNet121, DenseNet169, and DenseNet201. Finally, we quantitatively compared the benchmarked CNN architectures and their environment classification predictions using an operational metric called \“NetScore,\” which balances the image classification accuracy with the computational and memory storage requirements (i.e., important for onboard real-time inference with mobile computing devices). Our comparative analyses showed that the EfficientNetB0 network achieves the highest test accuracy; VGG16 the fastest inference time; and MobileNetV2 the best NetScore, which can inform the optimal architecture design or selection depending on the desired performance. Overall, this study provides a large-scale benchmark and reference for next-generation environment classification systems for robotic leg prostheses and exoskeletons.

}, keywords = {Artificial Intelligence, Biomechatronics, Computer Vision, Deep Learning, ExoNet, Exoskeletons, Prosthetics, Rehabilitation Robotics, wearables}, doi = {10.3389/fnbot.2021.730965}, url = {https://www.frontiersin.org/articles/10.3389/fnbot.2021.730965/full}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} } @inproceedings {893, title = {Comparative Analysis of Environment Recognition Systems for Control of Lower-Limb Exoskeletons and Prostheses}, booktitle = {IEEE RAS/EMBS International Conference for Biomedical Robotics and Biomechatronics (BioRob)}, year = {2020}, month = {10/2020}, publisher = {IEEE}, organization = {IEEE}, address = {New York City, NY, USA}, abstract = {

Environment recognition systems can facilitate the predictive control of lower-limb exoskeletons and prostheses by recognizing the oncoming walking environment prior to physical interactions. While many environment recognition systems have been developed using different wearable technology and classification algorithms, their relative operational performances have not been evaluated. Motivated to determine the state-of-the-science and propose future directions for research innovation, we conducted an extensive comparative analysis of the wearable technology, training datasets, and classification algorithms used for vision-based environment recognition. The advantages and drawbacks of different wearable cameras and training datasets were reviewed. Environment recognition systems using pattern recognition, machine learning, and convolutional neural networks for image classification were compared. We evaluated the performances of different deep learning networks using a novel balanced metric called \“NetScore\”, which considers the image classification accuracy, and computational and memory storage requirements. Based on our analysis, future research in environment recognition systems for lower-limb exoskeletons and prostheses should consider developing 1) efficient deep convolutional neural networks for onboard classification, and 2) large-scale open-source datasets for training and comparing image classification algorithms from different researchers.\ 

}, keywords = {Artificial Intelligence, Biomechatronics, Computer Vision, Environment Recognition, Exoskeletons, Image Classification, Prostheses, Rehabilitation, Wearable Technology}, doi = {10.1109/BioRob49111.2020.9224364}, url = {https://ieeexplore.ieee.org/document/9224364}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} } @article {906, title = {ExoNet Database: Wearable Camera Images of Human Locomotion Environments}, journal = {Frontiers in Robotics and Artificial Intelligence}, volume = {7}, year = {2020}, month = {12/2020}, abstract = {

Advances in computer vision and deep learning are allowing researchers to develop environment recognition systems for robotic exoskeletons and prostheses. However, small-scale and private training datasets have impeded the development and dissemination of image classification algorithms for classifying human walking environments. To address these limitations, we developed ExoNet - the first open-source, large-scale hierarchical database of high-resolution wearable camera images of human locomotion environments. Unparalleled in scale and diversity, ExoNet contains over 5.6 million images of indoor and outdoor real-world walking environments, which were collected using a lightweight wearable camera system throughout the summer, fall, and winter seasons. Approximately 923,000 images in ExoNet were human-annotated using a 12-class hierarchical labelling architecture. Available publicly through IEEE DataPort, ExoNet offers an unprecedented shared platform to train, develop, and compare next-generation image classification algorithms for human locomotion environment recognition. Besides the control of exoskeletons and prostheses, applications of ExoNet could extend to humanoids and autonomous legged\ robots.\ 

}, keywords = {Artificial Intelligence, Biomechatronics, Computer Vision, Deep Learning, Environment Recognition, Exoskeletons, Prosthetics, Rehabilitation Robotics, Wearable Technology}, doi = {10.3389/frobt.2020.562061}, url = {https://www.frontiersin.org/articles/10.3389/frobt.2020.562061/full}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} } @article {883, title = {Integration of Machine Learning with Dynamics and Control: From Autonomous Cars to Biomechatronics}, number = {7-8}, year = {2019}, chapter = {Fall 2019}, abstract = {

Researchers at the University of Waterloo are strategically combining artificial intelligence (AI) methods with dynamics and model-based control approaches, to solve real-world problems that are too difficult or computationally intensive to tackle with either AI or conventional methods on their own. Our goal is to combine the best features of machine learning and model-based control theories, in contrast to computer scientists that are seeking purely AI solutions to dynamics and control problems. This article presents our recent work towards this goal, with an emphasis on two very active research applications: Autonomous Cars and Biomechatronics.

}, keywords = {Artificial Intelligence, Autonomous Cars, Biomechatronics, Deep Learning, Exoskeletons, Robotics}, url = {http://csme-scgm.ca/sites/default/files/BULLETIN-FALL-2019.pdf}, author = {Arash Hashemi and Yuan Lin and William McNally and Brokoslaw Laschowski and Bryce Hosking and Alexander Wong and John McPhee} } @inproceedings {874, title = {Preliminary Design of an Environment Recognition System for Controlling Robotic Lower-Limb Prostheses and Exoskeletons}, booktitle = {IEEE International Conference on Rehabilitation Robotics (ICORR) }, year = {2019}, month = {07/2019}, publisher = {IEEE }, organization = {IEEE }, address = {Toronto, Canada}, abstract = {

Drawing inspiration from autonomous vehicles, using future environment information could improve the control of wearable biomechatronic devices for assisting human locomotion. To the authors knowledge, this research represents the first documented investigation using machine vision and deep convolutional neural networks for environment recognition to support the predictive control of robotic lower-limb prostheses and exoskeletons. One participant was instrumented with a battery-powered, chest-mounted RGB camera system. Approximately 10 hours of video footage were experimentally collected while ambulating throughout unknown outdoor and indoor environments. The sampled images were preprocessed and individually labelled. A deep convolutional neural network was developed and trained to automatically recognize three walking environments: level-ground, incline staircases, and decline staircases. The environment recognition system achieved 94.85\% overall image classification accuracy. Extending these preliminary findings, future research should incorporate other environment classes (e.g., incline ramps) and integrate the environment recognition system with electromechanical sensors and/or surface electromyography for automated locomotion mode recognition. The challenges associated with implementing deep learning on wearable biomechatronic devices are discussed.\ 

}, keywords = {Artificial Intelligence, Autonomous Control, Deep Learning, Exoskeleton, Machine Vision, Prostheses, Robotics}, doi = {10.1109/ICORR.2019.8779540}, url = {https://ieeexplore.ieee.org/document/8779540}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} }