@article {964, title = {Multibody Dynamics and Control using Machine Learning}, journal = {Multibody System Dynamics}, year = {2023}, keywords = {Data- driven modeling, Data-driven control, Deep Learning, Machine Learning, Model-based control, Multibody system}, author = {Arash Hashemi and Grzegorz Orzechowski and Aki Mikkola and John McPhee} } @article {919, title = {Environment Classification for Robotic Leg Prostheses and Exoskeletons using Deep Convolutional Neural Networks}, journal = {Frontiers in Neurorobotics}, volume = {15}, year = {2022}, month = {02/2022}, publisher = {bioRxiv}, abstract = {

Robotic leg prostheses and exoskeletons can provide powered locomotor assistance to older adults and/or persons with physical disabilities. However, the current locomotion mode recognition systems being developed for automated high-level control and decision-making rely on mechanical, inertial, and/or neuromuscular sensors, which inherently have limited prediction horizons (i.e., analogous to walking blindfolded). Inspired by the human vision-locomotor control system, we developed an environment classification system powered by computer vision and deep learning to predict the oncoming walking environments prior to physical interaction, therein allowing for more accurate and robust high-level control decisions. In this study, we first reviewed the development of our \“ExoNet\” database\—the largest and most diverse open-source dataset of wearable camera images of indoor and outdoor real-world walking environments, which were annotated using a hierarchical labeling architecture. We then trained and tested over a dozen state-of-the-art deep convolutional neural networks (CNNs) on the ExoNet database for image classification and automatic feature engineering, including: EfficientNetB0, InceptionV3, MobileNet, MobileNetV2, VGG16, VGG19, Xception, ResNet50, ResNet101, ResNet152, DenseNet121, DenseNet169, and DenseNet201. Finally, we quantitatively compared the benchmarked CNN architectures and their environment classification predictions using an operational metric called \“NetScore,\” which balances the image classification accuracy with the computational and memory storage requirements (i.e., important for onboard real-time inference with mobile computing devices). Our comparative analyses showed that the EfficientNetB0 network achieves the highest test accuracy; VGG16 the fastest inference time; and MobileNetV2 the best NetScore, which can inform the optimal architecture design or selection depending on the desired performance. Overall, this study provides a large-scale benchmark and reference for next-generation environment classification systems for robotic leg prostheses and exoskeletons.

}, keywords = {Artificial Intelligence, Biomechatronics, Computer Vision, Deep Learning, ExoNet, Exoskeletons, Prosthetics, Rehabilitation Robotics, wearables}, doi = {10.3389/fnbot.2021.730965}, url = {https://www.frontiersin.org/articles/10.3389/fnbot.2021.730965/full}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} } @inproceedings {915, title = {Computer Vision and Deep Learning for Environment-Adaptive Control of Robotic Lower-Limb Exoskeletons}, booktitle = {Annual International Conference of the IEEE Engineering in Medicine \& Biology Society (EMBC)}, year = {2021}, month = {04/2021}, publisher = {IEEE }, organization = {IEEE }, abstract = {

Robotic exoskeletons require human control and decision making to switch between different locomotion modes, which can be inconvenient and cognitively demanding. To support the development of automated locomotion mode recognition systems (i.e., high-level controllers), we designed an environment recognition system using computer vision and deep learning. We collected over 5.6 million images of indoor and outdoor real-world walking environments using a wearable camera system, of which ~923,000 images were annotated using a 12-class hierarchical labelling architecture (called the ExoNet database). We then trained and tested the EfficientNetB0 convolutional neural network, designed for efficiency using neural architecture search, to predict the different walking environments. Our environment recognition system achieved ~73\% image classification accuracy. While these preliminary results benchmark EfficientNetB0 on the ExoNet database, further research is needed to compare different image classification algorithms to develop an accurate and real-time environment-adaptive locomotion mode recognition system for robotic exoskeleton control.

}, keywords = {Biomechatronics, Computer Vision, Deep Learning, Exoskeletons, Rehabilitation, Robotics}, doi = {10.1109/EMBC46164.2021.9630064}, url = {https://ieeexplore.ieee.org/document/9630064}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} } @article {906, title = {ExoNet Database: Wearable Camera Images of Human Locomotion Environments}, journal = {Frontiers in Robotics and Artificial Intelligence}, volume = {7}, year = {2020}, month = {12/2020}, abstract = {

Advances in computer vision and deep learning are allowing researchers to develop environment recognition systems for robotic exoskeletons and prostheses. However, small-scale and private training datasets have impeded the development and dissemination of image classification algorithms for classifying human walking environments. To address these limitations, we developed ExoNet - the first open-source, large-scale hierarchical database of high-resolution wearable camera images of human locomotion environments. Unparalleled in scale and diversity, ExoNet contains over 5.6 million images of indoor and outdoor real-world walking environments, which were collected using a lightweight wearable camera system throughout the summer, fall, and winter seasons. Approximately 923,000 images in ExoNet were human-annotated using a 12-class hierarchical labelling architecture. Available publicly through IEEE DataPort, ExoNet offers an unprecedented shared platform to train, develop, and compare next-generation image classification algorithms for human locomotion environment recognition. Besides the control of exoskeletons and prostheses, applications of ExoNet could extend to humanoids and autonomous legged\ robots.\ 

}, keywords = {Artificial Intelligence, Biomechatronics, Computer Vision, Deep Learning, Environment Recognition, Exoskeletons, Prosthetics, Rehabilitation Robotics, Wearable Technology}, doi = {10.3389/frobt.2020.562061}, url = {https://www.frontiersin.org/articles/10.3389/frobt.2020.562061/full}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} } @article {883, title = {Integration of Machine Learning with Dynamics and Control: From Autonomous Cars to Biomechatronics}, number = {7-8}, year = {2019}, chapter = {Fall 2019}, abstract = {

Researchers at the University of Waterloo are strategically combining artificial intelligence (AI) methods with dynamics and model-based control approaches, to solve real-world problems that are too difficult or computationally intensive to tackle with either AI or conventional methods on their own. Our goal is to combine the best features of machine learning and model-based control theories, in contrast to computer scientists that are seeking purely AI solutions to dynamics and control problems. This article presents our recent work towards this goal, with an emphasis on two very active research applications: Autonomous Cars and Biomechatronics.

}, keywords = {Artificial Intelligence, Autonomous Cars, Biomechatronics, Deep Learning, Exoskeletons, Robotics}, url = {http://csme-scgm.ca/sites/default/files/BULLETIN-FALL-2019.pdf}, author = {Arash Hashemi and Yuan Lin and William McNally and Brokoslaw Laschowski and Bryce Hosking and Alexander Wong and John McPhee} } @inproceedings {874, title = {Preliminary Design of an Environment Recognition System for Controlling Robotic Lower-Limb Prostheses and Exoskeletons}, booktitle = {IEEE International Conference on Rehabilitation Robotics (ICORR) }, year = {2019}, month = {07/2019}, publisher = {IEEE }, organization = {IEEE }, address = {Toronto, Canada}, abstract = {

Drawing inspiration from autonomous vehicles, using future environment information could improve the control of wearable biomechatronic devices for assisting human locomotion. To the authors knowledge, this research represents the first documented investigation using machine vision and deep convolutional neural networks for environment recognition to support the predictive control of robotic lower-limb prostheses and exoskeletons. One participant was instrumented with a battery-powered, chest-mounted RGB camera system. Approximately 10 hours of video footage were experimentally collected while ambulating throughout unknown outdoor and indoor environments. The sampled images were preprocessed and individually labelled. A deep convolutional neural network was developed and trained to automatically recognize three walking environments: level-ground, incline staircases, and decline staircases. The environment recognition system achieved 94.85\% overall image classification accuracy. Extending these preliminary findings, future research should incorporate other environment classes (e.g., incline ramps) and integrate the environment recognition system with electromechanical sensors and/or surface electromyography for automated locomotion mode recognition. The challenges associated with implementing deep learning on wearable biomechatronic devices are discussed.\ 

}, keywords = {Artificial Intelligence, Autonomous Control, Deep Learning, Exoskeleton, Machine Vision, Prostheses, Robotics}, doi = {10.1109/ICORR.2019.8779540}, url = {https://ieeexplore.ieee.org/document/8779540}, author = {Brokoslaw Laschowski and William McNally and Alexander Wong and John McPhee} }