@conference {godaz2021acml, title = {Vector Transport Free Riemannian LBFGS for Optimization on Symmetric Positive Definite Matrix Manifolds}, booktitle = {Asian Conference on Machine Learning (ACML)}, year = {Accepted}, month = {November}, pages = {8}, address = {Virtual}, abstract = {This work concentrates on optimization on Riemannian manifolds. The Limited-memory Broyden{\textendash}Fletcher{\textendash}Goldfarb{\textendash}Shanno (LBFGS) algorithm is a commonly used quasi-Newton method for numerical optimization in Euclidean spaces. Riemannian LBFGS (RLBFGS) is an extension of this method to Riemannian manifolds. RLBFGS involves computationally expensive vector transports as well as unfolding recursions using adjoint vector transports. In this article, we propose two mappings in the tangent space using the inverse second root and Cholesky decomposition. These mappings make both vector transport and adjoint vector transport identity and therefore isometric. Identity vector transport makes RLBFGS less computationally expensive and its isometry is also very useful in convergence analysis of RLBFGS. Moreover, under the proposed mappings, the Riemannian metric reduces to Euclidean inner product, which is much less computationally expensive. We focus on the Symmetric Positive Definite (SPD) manifolds which are beneficial in various fields such as data science and statistics. This work opens a research opportunity for extension of the proposed mappings to other well-known manifolds.}, url = {http://www.acml-conf.org/2021/conference/accepted-papers/81/}, author = {Reza Godaz and Benyamin Ghojogh and Reshad Hosseini and Reza Monsefi and Fakhri Karray and Mark Crowley} } @article {poorheravi2021cvis, title = {Acceleration of Large Margin Metric Learning for Nearest Neighbor Classification Using Triplet Mining and Stratified Sampling}, journal = {Journal of Computational Vision and Imaging Systems}, volume = {6}, number = {1}, year = {2021}, month = {Jan.}, pages = {5}, abstract = {Metric learning is a technique in manifold learning to find a projection subspace for increasing and decreasing the inter- and intra-class variances, respectively. Some metric learning methods are based on triplet learning with anchor-positive-negative triplets. Large margin metric learning for nearest neighbor classification is one of the fundamental methods to do this. Recently, Siamese networks have been introduced with the triplet loss. Many triplet mining methods have been developed for Siamese nets; however, these techniques have not been applied on the triplets of large margin metric learning. In this work, inspired by the mining methods for Siamese nets, we propose several triplet mining techniques for large margin metric learning. Moreover, a hierarchical approach is proposed, for acceleration and scalability of optimization, where triplets are selected by stratified sampling in hierarchical hyper-spheres. We analyze the proposed methods on three publicly available datasets.}, doi = {10.15353/jcvis.v6i1.3534}, url = {https://openjournals.uwaterloo.ca/index.php/vsl/article/view/3534}, author = {Poorheravi*, Parisa and Ghojogh*, Benyamin and Gaudet, Vincent and Karray, Fakhri and Crowley, Mark} } @conference {bellinger2021canai, title = {Active Measure Reinforcement Learning for Observation Cost Minimization: A framework for minimizing measurement costs in reinforcement learning}, booktitle = {Canadian Conference on Artificial Intelligence}, series = {Lecture Notes in Computer Science (LNCS)}, year = {2021}, pages = {12}, publisher = {Springer}, organization = {Springer}, abstract = {Markov Decision Processes (MDP) with explicit measurement cost are a class of en- vironments in which the agent learns to maximize the costed return. Here, we define the costed return as the discounted sum of rewards minus the sum of the explicit cost of measuring the next state. The RL agent can freely explore the relationship between actions and rewards but is charged each time it measures the next state. Thus, an op- timal agent must learn a policy without making a large number of measurements. We propose the active measure RL framework (Amrl) as a solution to this novel class of problem, and contrast it with standard reinforcement learning under full observability and planning under partially observability. We demonstrate that Amrl-Q agents learn to shift from a reliance on costly measurements to exploiting a learned transition model in order to reduce the number of real-world measurements and achieve a higher costed return. Our results demonstrate the superiority of Amrl-Q over standard RL methods, Q-learning and Dyna-Q, and POMCP for planning under a POMDP in environments with explicit measurement costs.}, keywords = {chemistry, Machine-Learning, proj-deepchemrl, reinforcement-learning, showcase, year-in-review-2021}, author = {Colin Bellinger and Rory Coles and Mark Crowley and Isaac Tamblyn} } @conference {allada2021embc, title = {Analysis of Language Embeddings for Classification of Unstructured Pathology Reports}, booktitle = {International Conference of the IEEE Engineering in Medicine and Biology Society}, year = {2021}, month = {November}, pages = {4}, publisher = {IEEE}, organization = {IEEE}, abstract = {A pathology report is one of the most significant medical documents providing interpretive insights into the visual appearance of the patient{\textquoteright}s biopsy sample. In digital pathology, high-resolution images of tissue samples are stored along with pathology reports. Despite the valuable information that pathology reports hold, they are not used in any systematic manner to promote computational pathology. In this work, we focus on analyzing the reports, which are generally unstructured documents written in English with sophisticated and highly specialized medical terminology. We provide a comparative analysis of various embedding models like BioBERT, Clinical BioBERT, BioMed-RoBERTa and Term Frequency-Inverse Document Frequency (TF-IDF), a traditional NLP technique, as well as the combination of embeddings from pre-trained models with TF-IDF. Our results demonstrate the effectiveness of various word embedding techniques for pathology reports.}, keywords = {Deep Neural Networks, digital pathology, natural language processing, proj-digipath, year-in-review-2021}, author = {Aishwarya Krishna Allada and Yuanxin Wang and Veni Jindal and Morteza Babee and H.R. Tizhoosh and Mark Crowley} } @conference {sikaroudi2021icpr, title = {Batch-Incremental Triplet Sampling for Training Triplet Networks Using Bayesian Updating Theorem}, booktitle = {25th International Conference on Pattern Recognition (ICPR)}, year = {2021}, month = {January}, pages = {7}, publisher = {IEEE}, organization = {IEEE}, address = {Milan, Italy (virtual)}, abstract = {Variants of Triplet networks are robust entities for learning a discriminative embedding subspace. There exist different triplet mining approaches for selecting the most suitable training triplets. Some of these mining methods rely on the extreme distances between instances, and some others make use of sampling. However, sampling from stochastic distributions of data rather than sampling merely from the existing embedding instances can provide more discriminative information. In this work, we sample triplets from distributions of data rather than from existing instances. We consider a multivariate normal distribution for the embedding of each class. Using Bayesian updating and conjugate priors, we update the distributions of classes dynamically by receiving the new mini-batches of training data. The proposed triplet mining with Bayesian updating can be used with any triplet-based loss function, e.g., triplet-loss or Neighborhood Component Analysis (NCA) loss. Accordingly, Our triplet mining approaches are called Bayesian Updating Triplet (BUT) and Bayesian Updating NCA (BUNCA), depending on which loss function is being used. Experimental results on two public datasets, namely MNIST and histopathology colorectal cancer (CRC), substantiate the effectiveness of the proposed triplet mining method.}, keywords = {digital-pathology, manifold-learning, proj-digipath, triplet-mining, year-in-review-2021}, url = {https://ieeexplore.ieee.org/document/9412478}, author = {Milad Sikaroudi and Benyamin Ghojogh and Fakhri Karray and Mark Crowley and H. R. Tizhoosh} } @unpublished {ghojogh2021smc, title = {Generative Locally Linear Embedding}, year = {2021}, month = {October}, type = {Preprint}, abstract = {Locally Linear Embedding (LLE) is a nonlin- ear spectral dimensionality reduction and manifold learning method. It has two main steps which are linear reconstruc- tion and linear embedding of points in the input space and embedding space, respectively. In this work, we propose two novel generative versions of LLE, named Generative LLE (GLLE), whose linear reconstruction steps are stochastic rather than deterministic. GLLE assumes that every data point is caused by its linear reconstruction weights as latent factors. The proposed GLLE algorithms can generate various LLE embeddings stochastically while all the generated embeddings relate to the original LLE embedding. We propose two versions for stochastic linear reconstruction, one using expectation maximization and another with direct sampling from a derived distribution by optimization. The proposed GLLE methods are closely related to and inspired by variational inference, factor analysis, and probabilistic principal component analysis. Our simulations show that the proposed GLLE methods work effectively in unfolding and generating submanifolds of data.}, keywords = {embeddings, manifold-learning, representation-learning, year-in-review-2021}, url = {https://arxiv.org/abs/2104.01525}, author = {Benyamin Ghojogh and Ali Ghodsi and Fakhri Karray and Mark Crowley} } @article {ghojogh2021softimp, title = {Generative locally linear embedding: A module for manifold unfolding and visualization}, journal = {Software Impacts}, volume = {9}, number = {100105}, year = {2021}, pages = {3}, publisher = {Elsevier}, abstract = {Data often have nonlinear patterns in machine learning. One can unfold the nonlinear manifold of a dataset for low-dimensional visualization and feature extraction. Locally Linear Embedding (LLE) is a nonlinear spectral method for dimensionality reduction and manifold unfolding. It embeds data using the same linear reconstruction weights as in the input space. In this paper, we propose an open source module which not only implements LLE, but also includes implementations of two generative LLE algorithms whose linear reconstruction phases are stochastic. Using this module, one can generate as many manifold unfoldings as desired for data visualization or feature extraction.}, author = {Ghojogh, Benyamin and Ghodsi, Ali and Karray, Fakhri and Crowley, Mark} } @conference {akgun2021hri, title = {Integrating Affective Expressions into the Search and Rescue Context in order to Improve Non-Verbal Human-Robot Interaction}, booktitle = {Workshop on Exploring Applications for Autonomous Non-Verbal Human-Robot Interactions (HRI)}, year = {2021}, month = {March}, publisher = {ACM}, organization = {ACM}, address = {Virtual}, keywords = {affective communication, emotion-modeling, human-robot-interaction, search-and-rescue, year-in-review-2021}, url = {https://sites.google.com/view/non-verbal-hri-2021/home}, author = {Akgun, Sami Alperen and Ghafurian, Moojan and Crowley, Mark and Dautenhahn, Kerstin} } @conference {sikaroudi2021isbi, title = {Magnification Generalization for Histopathology Image Embedding}, booktitle = {IEEE International Symposium on Biomedical Imaging (ISBI)}, year = {2021}, month = {April}, pages = {5}, abstract = {Histopathology image embedding is an active research area in computer vision. Most of the embedding models exclusively concentrate on a specific magnification level. However, a useful task in histopathology embedding is to train an embedding space regardless of the magnification level. Two main approaches for tackling this goal are domain adaptation and domain generalization, where the target magnification levels may or may not be introduced to the model in training, respectively. Although magnification adaptation is a well-studied topic in the literature, this paper, to the best of our knowledge, is the first work on magnification generalization for histopathology image embedding. We use an episodic trainable domain generalization technique for magnification generalization, namely Model Agnostic Learning of Semantic Features (MASF), which works based on the Model Agnostic Meta-Learning (MAML) concept. Our experimental results on a breast cancer histopathology dataset with four different magnification levels show the proposed method{\textquoteright}s effectiveness for magnification generalization.}, keywords = {proj-digipath, year-in-review-2021}, author = {Milad Sikaroudi and Benyamin Ghojogh and Fakhri Karray and Mark Crowley and H. R. Tizhoosh} } @conference {ganapathi-subramanian2021aamas, title = {Partially Observable Mean Field Reinforcement Learning}, booktitle = {Proceedings of the 20th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)}, series = {AAMAS {\textquoteright}21}, year = {2021}, month = {3{\textendash}7 May}, pages = {537-545}, publisher = {International Foundation for Autonomous Agents and Multiagent Systems}, organization = {International Foundation for Autonomous Agents and Multiagent Systems}, address = {London, United Kingdom}, abstract = {Traditional multi-agent reinforcement learning algorithms are not scalable to environments with more than a few agents, since these algorithms are exponential in the number of agents. Recent research has introduced successful methods to scale multi-agent reinforcement learning algorithms to many agent scenarios using mean field theory. Previous work in this field assumes that an agent has access to exact cumulative metrics regarding the mean field behaviour of the system, which it can then use to take its actions. In this paper, we relax this assumption and maintain a distribution to model the uncertainty regarding the mean field of the system. We consider two different settings for this problem. In the first setting, only agents in a fixed neighbourhood are visible, while in the second setting, the visibility of agents is determined at random based on distances. For each of these settings, we introduce a Q-learning based algorithm that can learn effectively. We prove that this Q-learning estimate stays very close to the Nash Q-value (under a common set of assumptions) for the first setting. We also empirically show our algorithms outperform multiple baselines in three different games in the MAgents framework, which supports large environments with many agents learning simultaneously to achieve possibly distinct goals.}, keywords = {mean field theory, Multi-Agent Reinforcement Learning, partial observation, reinforcement learning, showcase, year-in-review-2021}, isbn = {9781450383073}, author = {Ganapathi Subramanian, Sriram and Taylor, Matthew and Crowley, Mark and Poupart, Pascal} } @conference {crowley2020coviddata, title = {Prediction and Causality: How Can Machine Learning be Used for COVID-19?}, booktitle = {"What Needs to be done in order to Curb the Spread of Covid-19: Exposure Notification, Legal Considerations, and Statistical Modeling", a Conference on Data and Privacy during a Global Pandemic}, year = {2021}, month = {July}, pages = {6}, publisher = {Master of Public Service (MPS) Policy and Data Lab, University of Waterloo}, organization = {Master of Public Service (MPS) Policy and Data Lab, University of Waterloo}, address = {Waterloo, Canada}, keywords = {Causality, covid-19, epidemiology, graphical-models, Machine-Learning, medical, reinforcement-learning, year-in-review-2021}, url = {https://uwaterloo.ca/master-of-public-service/events/data-and-privacy-during-global-pandemic-conference}, author = {Mark Crowley} } @article {ghojogh2021mlwajrnl, title = {Quantile\–Quantile Embedding for Distribution Transformation and Manifold Embedding with Ability to Choose the Embedding Distribution}, journal = {Machine Learning with Applications (MLWA)}, volume = {6}, year = {2021}, note = {1}, abstract = {We propose a new embedding method, named Quantile-Quantile Embedding (QQE), for distribution transformation and manifold embedding with the ability to choose the embedding distribution. QQE, which uses the concept of quantile-quantile plot from visual statistical tests, can transform the distribution of data to any theoretical desired distribution or empirical reference sample. Moreover, QQE gives the user a choice of embedding distribution in embedding the manifold of data into the low dimensional embedding space. It can also be used for modifying the embedding distribution of other dimensionality reduction methods, such as PCA, t-SNE, and deep metric learning, for better representation or visualization of data. We propose QQE in both unsupervised and supervised forms. QQE can also transform a distribution to either an exact reference distribution or its shape. We show that QQE allows for better discrimination of classes in some cases. Our experiments on different synthetic and image datasets show the effectiveness of the proposed embedding method.}, keywords = {dimensionality-reduction, embeddings, image-processing, image-understanding, manifold-learning, proj-digipath, showcase, year-in-review-2021}, doi = {https://doi.org/10.1016/j.mlwa.2021.100088}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @conference {ghafurian2020interact, title = {Recognition of a Robot{\textquoteright}s Affective Expressions under Conditions with Limited Visibility}, booktitle = {18th International Conference promoted by the IFIP Technical Committee 13 on Human{\textendash}Computer Interaction (INTERACT 2021)}, year = {2021}, month = {September}, pages = {22}, address = {Bari, Italy}, abstract = {The capability of showing affective expressions is important for the design of social robots in many contexts, especially where the robot is designed to communicate with humans. It is reasonable to expect that, similar to all other interaction modalities, communicating with affective expressions is not without limitations. In this paper, we present two online video studies (N=72 and N=50) and investigate if/how much the recognition of affective displays of a zoomorphic robot is affected under situations with different levels of visibility. Five affective expressions combined with five visibility effects were studied. The intensity of the effects was more pronounced in the second experiment. While visual constraints affected recognition of expressions, our results showed that affective displays of the robot conveyed through its head and body gestures can be robust and recognition rates can still be high even under severe constraints. This study supported the effectiveness of using affective displays as a complementary communication modality in human-robot interaction in situations with low visibility.}, keywords = {human-robot-interaction, showcase, year-in-review-2021}, author = {Moojan Ghafurian and Sami Alperen Akgun and Mark Crowley and Kerstin Dautenhahn} } @conference {ghojogh2020anomaly, title = {Anomaly Detection and Prototype Selection Using Polyhedron Curvature}, booktitle = {Canadian Conference on Artificial Intelligence}, year = {2020}, pages = {10}, publisher = {Springer}, organization = {Springer}, address = {Ottawa, Canada}, keywords = {anomaly detection, representation}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @conference {ghojogh2020backprojection, title = {Backprojection for Training Feedforward Neural Networks in the Input and Feature Spaces}, booktitle = {International Conference on Image Analysis and Recognition}, year = {2020}, publisher = {Springer}, organization = {Springer}, address = {P{\'o}voa de Varzim, Portugal (virtual)}, keywords = {Deep Neural Networks, neural networks}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @conference {24122, title = {Deep Multi Agent Reinforcement Learning for Autonomous Driving}, booktitle = {Canadian Conference on Artificial Intelligence}, volume = {LNAI 12109}, year = {2020}, pages = {17}, publisher = {Spring, Lecture Notes in Artificial Intelligence}, organization = {Spring, Lecture Notes in Artificial Intelligence}, abstract = {

Deep Learning and back-propagation have been successfully used to perform centralized training with communication protocols among multiple agents in a cooperative environment. In this work, we present techniques for centralized training of Multi-Agent Deep Reinforcement Learning (MARL) using the model-free Deep Q-Network (DQN) as the baseline model and communication between agents. We present two novel, scalable and centralized MARL training techniques (MA-MeSN, MA- BoN), which achieve faster convergence and higher cumulative reward in complex domains like autonomous driving simulators. Subsequently, we present a memory module to achieve a decentralized cooperative pol- icy for execution and thus addressing the challenges of noise and com- munication bottlenecks in real-time communication channels. This work theoretically and empirically compares our centralized and decentralized training algorithms to current research in the field of MARL. We also present and release a new OpenAI-Gym environment which can be used for multi-agent research as it simulates multiple autonomous cars driving on a highway. We compare the performance of our centralized algorithms to existing state-of-the-art algorithms, DIAL and IMS based on cumu- lative reward achieved per episode. MA-MeSN and MA-BoN achieve a cumulative reward of at least 263\% of the reward achieved by the DIAL and IMS. We also present an ablation study of the scalability of MA- BoN showing that it has a linear time and space complexity compared to quadratic for DIAL in the number of agents.

}, author = {Sushrut Bhalla and Sriram Ganapathi Subramanian and Mark Crowley} } @proceedings {basiri2020distributed, title = {Distributed Nonlinear Model Predictive Control and Metric Learning for Heterogeneous Vehicle Platooning with Cut-in/Cut-out Maneuvers}, journal = {59th IEEE Conference on Decision and Control (CDC) 2020}, year = {2020}, url = {arXiv preprint arXiv:2004.00417}, author = {Basiri, Mohammad Hossein and Ghojogh, Benyamin and Azad, Nasser L and Fischmeister, Sebastian and Karray, Fakhri and Crowley, Mark} } @conference {ghojogh2020fisher, title = {Fisher Discriminant Triplet and Contrastive Losses for Training Siamese Networks}, booktitle = {IEEE International Joint Conference on Neural Networks (IJCNN)}, year = {2020}, publisher = {IEEE}, organization = {IEEE}, address = {Glasgow, UK}, author = {Ghojogh, Benyamin and Sikaroudi, Milad and Shafiei, Sobhan and Tizhoosh, H.R. and Karray, Fakhri and Crowley, Mark} } @conference {ghojogh2019rda, title = {Generalized Subspace Learning by Roweis Discriminant Analysis}, booktitle = {International Conference on Image Analysis and Recognition}, year = {2020}, publisher = {Springer}, organization = {Springer}, address = {P{\'o}voa de Varzim, Portugal (virtual)}, abstract = {We present a new method which generalizes subspace learning based on eigenvalue and generalized eigenvalue problems. This method, Roweis Discriminant Analysis (RDA), is named after Sam Roweis to whom the field of subspace learning owes significantly. RDA is a family of infinite number of algorithms where Principal Component Analysis (PCA), Supervised PCA (SPCA), and Fisher Discriminant Analysis (FDA) are special cases. One of the extreme special cases, which we name Double Supervised Discriminant Analysis (DSDA), uses the labels twice; it is novel and has not appeared elsewhere. We propose a dual for RDA for some special cases. We also propose kernel RDA, generalizing kernel PCA, kernel SPCA, and kernel FDA, using both dual RDA and representation theory. Our theoretical analysis explains previously known facts such as why SPCA can use regression but FDA cannot, why PCA and SPCA have duals but FDA does not, why kernel PCA and kernel SPCA use kernel trick but kernel FDA does not, and why PCA is the best linear method for reconstruction. Roweisfaces and kernel Roweisfaces are also proposed generalizing eigenfaces, Fisherfaces, supervised eigenfaces, and their kernel variants. We also report experiments showing the effectiveness of RDA and kernel RDA on some benchmark datasets.}, keywords = {Data Reduction, Manifold learning, Numerosity Reduction}, url = {http://arxiv.org/abs/1910.05437}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @proceedings {ma2020isolation, title = {Isolation Mondrian Forest for Batch and Online Anomaly Detection}, journal = {IEEE International Conference on Systems, Man, and Cybernetics (SMC) 2020}, year = {2020}, abstract = {

We propose a new method, named isolation Mon- drian forest (iMondrian forest), for batch and online anomaly detection. The proposed method is a novel hybrid of isolation forest and Mondrian forest which are existing methods for batch anomaly detection and online random forest, respectively. iMondrian forest takes the idea of isolation, using the depth of a node in a tree, and implements it in the Mondrian forest structure. The result is a new data structure which can accept streaming data in an online manner while being used for anomaly detection. Our experiments show that iMondrian forest mostly performs better than isolation forest in batch settings and has better or comparable performance against other batch and online anomaly detection methods.

}, url = {arXiv preprint arXiv:2003.03692}, author = {Ma, Haoran and Ghojogh, Benyamin and Samad, Maria N and Zheng, Dongyu and Crowley, Mark} } @conference {subramanian2020multi, title = {Multi Type Mean Field Reinforcement Learning}, booktitle = {AAMAS 2020}, year = {2020}, abstract = {Mean field theory provides an effective way of scaling multiagent reinforcement learning algorithms to environments with many agents that can be abstracted by a virtual mean agent. In this paper, we extend mean field multiagent algorithms to multiple types. The types enable the relaxation of a core assumption in mean field games, which is that all agents in the environment are playing almost similar strategies and have the same goal. We conduct experiments on three different testbeds for the field of many agent reinforcement learning, based on the standard MAgents framework. We consider two different kinds of mean field games: a) Games where agents belong to predefined types that are known a priori and b) Games where the type of each agent is unknown and therefore must be learned based on observations. We introduce new algorithms for each type of game and demonstrate their superior performance over state of the art algorithms that assume that all agents belong to the same type and other baseline algorithms in the MAgent framework}, keywords = {mean-field-reinforcement-learning, proj-meanfield, reinforcement-learning}, url = {https://www.semanticscholar.org/paper/Multi-Type-Mean-Field-Reinforcement-Learning-Subramanian-Poupart/9a32f1527193f0cc753acc0606886a28ede95c8e}, author = {Sriram Ganapathi Subramanian and Pascal Poupart and Matthew E. Taylor and Nidhi Hegde} } @patent {29524, title = {Multi-Level Collaborative Control System With Dual Neural Network Planning For Autonomous Vehicle Control In A Noisy Environment}, volume = {US11131992B2}, year = {2020}, edition = {United States of America}, chapter = {US}, abstract = {

[[{"fid":"8001","view_mode":"default","type":"media","attributes":{"height":"1346","width":"1875","style":"width: 600px; height: 431px;","alt":"Design schematic for patent","title":"Design schematic for patent","class":"media-element file-default"}}]]

A RLP system for a host vehicle includes a memory and levels. The memory stores a RLP algorithm, which is a multi-agent collaborative DQN with PER algorithm. A first level includes a data processing module that provides sensor data, object location data, and state information of the host vehicle and other vehicles. A second level includes a coordinate location module that, based on the sensor data, the object location data, the state information, and a refined policy provided by the third level, generates an updated policy and a set of future coordinate locations implemented via the first level. A third level includes evaluation and target neural networks and a processor that executes instructions of the RLP algorithm for collaborative action planning between the host and other vehicles based on outputs of the evaluation and target networks and to generate the refined policy based on reward values associated with events.

}, url = {https://patents.google.com/patent/US11131992B2/en}, author = {Du, Zhiyuan and Lull, Joseph and Malhan, Rajesh and Ganapathi Subramanian, Sriram and Bhalla, Sushrut and Sambee, Jaspreet and Crowley, Mark and Fischmeister, Sebastian and Shin, Donghyun and Melek, William and Fidan, Baris and Woo, Ami and Sahoo, Bismaya} } @conference {sikaroudi-iscv-2020, title = {Offline versus Online Triplet Mining based on Extreme Distances of Histopathology Patches}, booktitle = {International Conference on Intelligent Systems and Computer Vision (ISCV 2020) }, year = {2020}, pages = {8}, publisher = {IEEE}, organization = {IEEE}, address = {Fez-Morrocco (virtual)}, abstract = {We analyze the effect of offline and online triplet mining for colorectal cancer (CRC) histopathology dataset containing 100,000 patches. We consider the extreme, i.e., farthest and nearest patches with respect to a given anchor, both in online and offline mining. While many works focus solely on how to select the triplets online (batch-wise), we also study the effect of extreme distances and neighbor patches before training in an offline fashion. We analyze the impacts of extreme cases for offline versus online mining, including easy positive, batch semi-hard, and batch hard triplet mining as well as the neighborhood component analysis loss, its proxy version, and distance weighted sampling. We also investigate online approaches based on extreme distance and comprehensively compare the performance of offline and online mining based on the data patterns and explain offline mining as a tractable generalization of the online mining with large mini-batch size. As well, we discuss the relations of different colorectal tissue types in terms of extreme distances. We found that offline mining can generate a better statistical representation of the population by working on the whole dataset.}, keywords = {digital pathology, machine learning, triplet mining}, url = {https://arxiv.org/abs/2007.02200}, author = {Milad Sikaroudi and Benyamin Ghojogh and Amir Safarpoor and Fakhri Karray and Mark Crowley and H. R. Tizhoosh} } @article {25888, title = {A review of machine learning applications in wildfire science and management.}, journal = {Environmental Reviews}, volume = {28}, year = {2020}, pages = {73}, abstract = {Artificial intelligence has been applied in wildfire science and management since the 1990s, with early applications including neural networks and expert systems. Since then the field has rapidly progressed congruently with the wide adoption of machine learning (ML) methods in the environmental sciences. Here, we present a scoping review of ML applications in wildfire science and management. Our overall objective is to improve awareness of ML methods among wildfire researchers and managers, as well as illustrate the diverse and challenging range of problems in wildfire science available to ML data scientists. To that end, we first present an overview of popular ML approaches used in wildfire science to date, and then review the use of ML in wildfire science as broadly categorized into six problem domains, including: 1) fuels characterization, fire detection, and mapping; 2) fire weather and climate change; 3) fire occurrence, susceptibility, and risk; 4) fire behavior prediction; 5) fire effects; and 6) fire management. Furthermore, we discuss the advantages and limitations of various ML approaches relating to data size, computational requirements, generalizability, and interpretability, as well as identify opportunities for future advances in the science and management of wildfires within a data science context. In total, we identfied 300 relevant publications up to the end of 2019, where the most frequently used ML methods across problem domains included random forests, MaxEnt, artificial neural networks, decision trees, support vector machines, and genetic algorithms. As such, there exists opportunities to apply more current ML methods {\textemdash} including deep learning and agent based learning {\textemdash} in the wildfire sciences, especially in instances involving very large multivariate datasets. We must recognize, however, that despite the ability of ML methods to learn on their own, expertise in wildfire science is necessary to ensure realistic modelling of fire processes across multiple scales, while the complexity of some ML methods, such as deep learning, requires a dedicated and sophisticated knowledge of their application. Finally, we stress that the wildfire research and management communities play an active role in providing relevant, high quality, and freely available wildfire data for use by practitioners of ML methods.}, url = {https://www.nrcresearchpress.com/doi/10.1139/er-2020-0019$\#$.X1jbKtNKhTY}, author = {Coogan, Sean CP and Jain, Piyush and Ganapathi Subramanian, Sriram and Taylor, Steve and Crowley, Mark and Flannigan, Mike D} } @conference {sikaroudi2020supervision, title = {Supervision and Source Domain Impact on Representation Learning: A Histopathology Case Study}, booktitle = {International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC{\textquoteright}20)}, year = {2020}, publisher = {IEEE Engineering in Medicine and Biology Society}, organization = {IEEE Engineering in Medicine and Biology Society}, address = {42nd Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC{\textquoteright}20)}, abstract = {

As many algorithms depend on a suitable representation of data, learning unique features is considered a crucial task. Although supervised techniques using deep neural networks have boosted the performance of representation learning, the need for a large set of labeled data limits the application of such methods. As an example, high-quality delineations of regions of interest in the field of pathology is a tedious and time-consuming task due to the large image dimensions. In this work, we explored the performance of a deep neural network and triplet loss in the area of representation learning. We investigated the notion of similarity and dissimilarity in pathology whole-slide images and compared different setups from unsupervised and semi-supervised to supervised learning in our experiments. Additionally, different approaches were tested, applying few-shot learning on two publicly available pathology image datasets. We achieved high accuracy and generalization when the learned representations were applied to two different pathology datasets.

}, keywords = {digipath, digital pathology, histopathology, medical, representation learning}, url = {https://embs.papercept.net/conferences/scripts/rtf/EMBC20_ContentListWeb_1.html$\#$moat2-15_02}, author = {Sikaroudi, Milad and Safarpoor, Amir and Ghojogh, Benyamin and Shafiei, Sobhan and Crowley, Mark and Tizhoosh, HR} } @conference {ghojogh2020theoretical, title = {Theoretical Insights into the Use of Structural Similarity Index In Generative Models and Inferential Autoencoders}, booktitle = {International Conference on Image Analysis and Recognition}, year = {2020}, publisher = {Springer}, organization = {Springer}, address = {P{\'o}voa de Varzim, Portugal (virtual)}, keywords = {image understanding, Manifold learning, representation}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @conference {akgun-icmi-2020, title = {Using Emotions to Complement Multi-Modal Human-Robot Interaction in Urban Search and Rescue Scenarios}, booktitle = {22nd International Conference on Multimodal Interaction (ICMI-2020)}, year = {2020}, month = {October}, pages = {9}, address = {Utrecht, the Netherlands}, abstract = {In this study, we investigated if there is consensus in using emotions as an additional communication channel to support multi-modal Human-Robot Interactions (HRI) in urban search and rescue scenarios, in which effectiveness of interactions is the key to success. The natural mappings between specific situations that might happen during missions and the proper emotions for the robots to show in that situation were obtained through a Mturk study with 78 participants. Suggested mappings for a set of 10 common situations were provided, which can be used to improve HRI in urban search and rescue situations.}, keywords = {disaster-response, multimodel interaction, robotics}, author = {Akgun, Sami Alperen and Ghafurian, Moojan and Crowley, Mark and Dautenhahn, Kerstin} } @conference {ghojogh2020weighted, title = {Weighted Fisher Discriminant Analysisin the Input and Feature Spaces}, booktitle = {International Conference on Image Analysis and Recognition (ICIAR)}, year = {2020}, publisher = {Springer}, organization = {Springer}, address = {P{\'o}voa de Varzim, Portugal (virtual)}, keywords = {Manifold learning}, author = {Ghojogh, Benyamin and Sikaroudi, Milad and Tizhoosh, H.R. and Karray, Fakhri and Crowley, Mark} } @unpublished {ghojogh2019addressing, title = {Addressing the Mystery of Population Decline of the Rose-Crested Blue Pipit in a Nature Preserve using Data Visualization}, journal = {ArXiv Preprint. ArXiv: 1903.06671}, year = {2019}, author = {Ghojogh, Benyamin and Crowley, Mark and Karray, Fakhri} } @conference {14923, title = {Artificial Counselor System For Stock Investment}, booktitle = {Innovative Applications of Artificial Intelligence (IAAI-19)}, year = {2019}, month = {27 January }, pages = {8}, publisher = {AAAI Press.}, organization = {AAAI Press.}, address = {IAAI-19 Conference, Honolulu, Hawaii, USA, 2019.}, abstract = {This paper proposes a novel trading system which plays the role of an artificial counselor for stock investment. In this paper, the stock future prices (technical features) are predicted using Support Vector Regression from the previous stock prices, average directional index, and parabolic stop and reverse index. Thereafter, the predicted prices are used to recommend which portions of the budget an investor should invest in different existing stocks to have an optimum expected profit considering their level of risk tolerance. Two different methods are used for suggesting best portions, which are Markowitz portfolio theory and fuzzy investment counselor. The first approach is an optimization-based method which considers merely technical features, while the second approach is based on Fuzzy Logic taking into account both technical and fundamental features of the stock market. The utilized dataset in this work is the New York Stock Exchange (NYSE). The experiments are performed on price prediction as well as optimum investment in various well-known stocks for a period of 30 days, and the results show the effectiveness of the proposed system.}, url = {https://aaai.org/ojs/index.php/AAAI/article/view/5016}, author = {Hadi NekoeiQachkanloo and Benyamin Ghojogh and Ali Saheb Pasand and Mark Crowley} } @conference {18089, title = {Compact Representation of a Multi-dimensional Combustion Manifold Using Deep Neural Networks}, booktitle = {European Conference on Machine Learning}, year = {2019}, pages = {8}, address = {Wurzburg, Germany}, abstract = {

[[{"fid":"4045","view_mode":"default","type":"media","attributes":{"height":"465","width":"538","style":"float: right; width: 250px; height: 216px; border-width: 1px; border-style: solid; margin-left: 10px; margin-right: 10px;","alt":"Example of Flamelet model","title":"Example of Flamelet model","class":"media-element file-default"}}]]The computational challenges in turbulent combustion simulations stem from the physical complexities and multi-scale nature of the problem which make it intractable to compute scale-resolving simulations. For most engineering applications, the large scale separation between the flame (typically sub-millimeter scale) and the characteristic turbulent flow (typically centimeter or meter scale) \ allows us to evoke simplifying assumptions--such as done for the flamelet model--to pre-compute all the chemical reactions and map them to a low-order manifold. The resulting manifold is then tabulated and looked-up at run-time. As the physical complexity of combustion simulations increases (including radiation, soot formation, pressure variations etc.) the dimensionality of the resulting manifold grows which impedes an efficient tabulation and look-up. In this paper we present a novel approach to model the multi-dimensional combustion manifold. We approximate the combustion manifold using a neural network function approximator and use it to predict the temperature and composition of the reaction. We present a novel training procedure which is developed to generate a smooth output curve for temperature over the course of a reaction. We then evaluate our work against the current approach of tabulation with linear interpolation in combustion simulations. We also provide an ablation study of our training procedure in the context of over-fitting in our model. The combustion dataset used for the modeling of combustion of H2 and O2 in this work is
released alongside this paper. See the poster version here.

[[{"fid":"4044","view_mode":"default","type":"media","attributes":{"style":"margin-left: 10px; margin-right: 10px; width: 830px; height: 266px;","alt":"Prediction Results","title":"Prediction Results","class":"media-element file-default "}}]]

}, author = {Sushrut Bhalla and Matthew Yao and Jean-Pierre Hickey and Mark Crowley} } @conference {18265, title = {Comparison of Deep Learning models for Determining Road Surface Condition from Roadside Camera Images and Weather Data}, booktitle = {The Transportation Association of Canada and Intelligent Transportation Systems Canada Joint Conference (TAC-ITS)}, year = {2019}, pages = {16}, address = {Halifax, Canada}, abstract = {

[[{"fid":"4043","view_mode":"default","type":"media","attributes":{"style":"float: right; width: 700px; height: 269px;","alt":"Example Image of Levels of Snow Road Cover","title":"Example Image of Levels of Snow Road Cover","class":"media-element file-default "}}]]Road maintenance during the Winter season is a safety critical and resource demanding operation. One of its key activities is determining road surface condition (RSC) in order to prioritize roads and allocate cleaning efforts such as plowing or salting. Two conventional approaches for determining RSC are: visual examination of roadside camera images by trained personnel and patrolling the roads to perform on-site inspections. However, with more than 500 cameras collecting images across Ontario, visual examination becomes a resource-intensive activity, difficult to scale especially during periods of snowstorms. This paper presents the results of a study focused on improving the efficiency of road maintenance operations. We use multiple Deep Learning models to automatically determine RSC from roadside camera images and weather variables, extending previous research where similar methods have been used to deal with the problem. The dataset we use was collected during the 2017-2018 Winter season from 40 stations connected to the Ontario Road Weather Information System (RWIS), it includes 14.000 labeled images and 70.000 weather measurements. We train and evaluate the performance of seven state-of-the-art models from the Computer Vision literature, including the recent DenseNet, NASNet, and MobileNet. Also, by integrating observations from weather variables, the models are able to better ascertain RSC under poor visibility conditions.

}, author = {Carrillo, Juan and Crowley, Mark and Pan, Guangyuan and Fu, Liping} } @conference {carrillo2019tac, title = {Comparison of Deep Learning models for Determining Road Surface Condition from Roadside Camera Images and Weather Data}, booktitle = {TAC-ITS Canada Joint Conference}, year = {2019}, pages = {17}, address = {Halifax, Canada}, abstract = {Road maintenance during the Winter season is a safety critical and resource demanding operation. One of its key activities is determining road surface condition (RSC) in order to prioritize roads and allocate cleaning efforts such as plowing or salting. Two conventional approaches for determining RSC are: visual examination of roadside camera images by trained personnel and patrolling the roads to perform on-site inspections. However, with more than 500 cameras collecting images across Ontario, visual examination becomes a resource-intensive activity, difficult to scale especially during periods of snow storms. This paper presents the preliminary results of an ongoing study focused on improving the efficiency of road maintenance operations. We use multiple Deep Learning models to automatically determine RSC from roadside camera images and weather variables, extending previous research where similar methods have been used to deal with the problem. The dataset we use was collected during the 2017-2018 Winter season from 40 stations connected to the Ontario Road Weather Information System (RWIS), it includes 14.000 labelled images and 70.000 weather measurements. In particular, we train and evaluate the performance of seven state-of-the-art models from the Computer Vision literature, including the recent DenseNet, NASNet, and MobileNet. Also, by integrating observations from weather variables, the models are able to better ascertain RSC under poor visibility conditions.}, keywords = {machine learning, smart cities}, url = {https://tac-its.ca/conference-papers/comparison-deep-learning-models-determining-road-surface-condition-roadside-camera}, author = {Carrillo, J. and Crowley, M. and Pan, G. and Fu, L.} } @unpublished {ghojogh2019distributed, title = {Distributed Voting in Beep Model}, journal = {arXiv preprint arXiv:1910.09882}, year = {2019}, author = {Ghojogh, Benyamin and Salehkaleybar, Saber} } @unpublished {ghojogh2019eigenvalue, title = {Eigenvalue and Generalized Eigenvalue Problems: Tutorial}, journal = {ArXiv Preprint arXiv:1903.11240}, year = {2019}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @unpublished {ghojogh2019feature, title = {Feature selection and feature extraction in pattern analysis: A literature review}, journal = {arXiv preprint arXiv:1905.02845}, year = {2019}, author = {Ghojogh, Benyamin and Samad, Maria N and Mashhadi, Sayema Asif and Kapoor, Tania and Ali, Wahab and Karray, Fakhri and Crowley, Mark} } @unpublished {ghojogh2019fisher, title = {Fisher and kernel fisher discriminant analysis: Tutorial}, journal = {arXiv preprint arXiv:1906.09436}, year = {2019}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @unpublished {ghojogh2019fitting, title = {Fitting A Mixture Distribution to Data: Tutorial}, journal = {ArXiv preprint. arXiv:1901.06708}, year = {2019}, keywords = {parameter estimation}, author = {Ghojogh, Benyamin and Ghojogh, Aydin and Crowley, Mark and Karray, Fakhri} } @unpublished {ghojogh2019hidden, title = {Hidden Markov Model: Tutorial}, year = {2019}, publisher = {engrXiv}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @conference {ghojogh2019image, title = {Image Structure Subspace Learning Using Structural Similarity Index}, booktitle = {International Conference on Image Analysis and Recognition (ICIAR-19)}, year = {2019}, pages = {33{\textendash}44}, publisher = {Springer, Cham}, organization = {Springer, Cham}, address = {Waterloo, Canada}, keywords = {Data Reduction, Manifold learning, Numerosity Reduction}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @conference {16828, title = {Instance Ranking and Numerosity Reduction Using Matrix Decompositionand Subspace Learning}, booktitle = {Canadian Conference on Artificial Intelligence}, year = {2019}, pages = {12}, publisher = {Springer{\textquoteright}s Lecture Notes in Artificial Intelligence.}, organization = {Springer{\textquoteright}s Lecture Notes in Artificial Intelligence.}, address = {Kingston, ON, Canada}, abstract = {

One way to deal with the ever increasing amount of available data for processing is to rank data instances by usefulness and reduce the dataset size. In this work, we introduce a framework to achieve this using matrix decomposition and subspace learning. Our central contribution is a novel similarity measure for data instances that uses the basis obtained from matrix decomposition of the dataset. Using this similarity measure, we propose several related algorithms for ranking data instances and performing numerosity reduction. We then validate the effectiveness of these algorithms for data reduction on several datasets for classification, regression, and clustering tasks.

}, author = {Benyamin Ghojogh and Mark Crowley} } @conference {15919, title = {Integration of Roadside Camera Images and Weather Data for monitoring Winter Road Surface Conditions}, booktitle = {Canadian Association of Road Safety Professionals CARSP Conference}, year = {2019}, pages = {4 (Won best paper award!)}, address = {CARSP Conference, Calgary, Alberta. }, abstract = {

This won the best paper award!

Background/Context: During the Winter season, real-time monitoring of road surface conditions is critical for the safety of drivers and road maintenance operations. Previous research has evaluated the potential of image classification methods for detecting road snow coverage by processing images from roadside cameras installed in RWIS (Road Weather Information System) stations. However, it is a challenging task due to limitations such as image resolution, camera angle, and illumination. Two common approaches to improve the accuracy of image classification methods are: adding more input features to the model and increasing the number of samples in the training dataset. Additional input features can be weather variables and more sample images can be added by including other roadside cameras. Although RWIS stations are equipped with both cameras and weather measurement instruments, they are only a subset of the total number of roadside cameras installed across transportation networks, most of which do not have weather measurement instruments. Thus, improvements in use of image data could benefit from additional data sources.

Aims/Objectives: The first objective of this study is to complete an exploratory data analysis over three data sources in Ontario: RWIS stations, all the other MTO (Ministry of Transportation of Ontario) roadside cameras, and Environment Canada weather stations. The second objective is to determine the feasibility of integrating these three datasets into a more extensive and richer dataset with weather variables as additional features and other MTO roadside cameras as additional sources of images.

Methods/Targets: First, we quantify the advantage of adding other MTO roadside cameras using spatial statistics, the number of monitored roads, and the coverage of ecoregions with different climate regimes. We then analyze experimental variograms from the literature and determine the feasibility of using Environment Canada stations and RWIS stations to interpolate weather variables for all the other MTO roadside cameras without weather instruments.

Results/Activities: By adding all other MTO cameras as image data sources, the total number of cameras in the dataset increases from 139 to 578 across Ontario. The average distance to the nearest camera decreases from 38.4km to 9.4km, and the number of monitored roads increases approximately four times. Additionally, six times more cameras are available in the four most populated ecoregions in Ontario. The experimental variograms show that it is feasible to interpolate weather variables with reasonable accuracy. Moreover, observations in the three datasets are collected with similar frequency, which facilitates our data integration approach.

Discussion/Deliverables: Integrating these three datasets is feasible and can benefit the design and development of automated image classification methods for monitoring road snow coverage. We do not consider data from pavement-embedded sensors, an additional line of research may explore the integration of this data. Our approach can provide actionable insights which can be used to more selectively perform manual patrolling to better identify road surface conditions.

Conclusions: Our initial results are promising and demonstrate that additional, image only datasets can be added to road monitoring data by using existing multimodal sensors as ground truth, which will lead to greater performance on the future image classification tasks.

}, url = {http://www.carsp.ca/research/research-papers/research-papers-search/download-info/integration-of-roadside-camera-images-and-weather-data-for-monitoring-winter-road-surface-conditions/}, author = {Juan Carrillo and Mark Crowley} } @conference {18266, title = {Learning Multi-Agent Communication with Reinforcement Learning}, booktitle = {Conference on Reinforcement Learning and Decision Making (RLDM-19)}, year = {2019}, pages = {4}, address = {Montreal, Canada.}, author = {Ganapthi Subramanian, Sriram and Bhalla, Sushrut and Crowley, Mark} } @unpublished {ghojogh2019linear, title = {Linear and Quadratic Discriminant Analysis: Tutorial}, journal = {arXiv preprint arXiv:1906.02590}, year = {2019}, author = {Ghojogh, Benyamin and Crowley, Mark} } @conference {ghojogh2019locally, title = {Locally Linear Image Structural Embedding for Image Structure Manifold Learning}, booktitle = {International Conference on Image Analysis and Recognition (ICIAR-19)}, year = {2019}, pages = {126{\textendash}138}, publisher = {Springer, Cham}, organization = {Springer, Cham}, address = {Waterloo, Canada}, keywords = {Data Reduction, Manifold learning, Numerosity Reduction}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @conference {ghojogh2019principal, title = {Principal Component Analysis Using Structural Similarity Index for Images}, booktitle = {International Conference on Image Analysis and Recognition (ICIAR-19)}, year = {2019}, pages = {77{\textendash}88}, publisher = {Springer, Cham}, organization = {Springer, Cham}, address = {Waterloo, Canada}, keywords = {Data Reduction, Manifold learning, Numerosity Reduction}, author = {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark} } @unpublished {ghojogh2019quantized, title = {Quantized Fisher Discriminant Analysis}, journal = {arXiv preprint arXiv:1909.03037}, year = {2019}, author = {Ghojogh, Benyamin and Pasand, Ali Saheb and Karray, Fakhri and Crowley, Mark} } @conference {CarrilloSciknow2019, title = {Semantic Workflows and Machine Learning for the Assessment of Carbon Storage by Urban Trees}, booktitle = {Third International Workshop on Capturing Scientific Knowledge (SciKnow19)}, year = {2019}, note = {1}, pages = {1{\textendash}6}, address = {Los Angeles, California, USA}, abstract = {Climate science is critical for understanding both the causes and consequences of changes in global temperatures and has become imperative for decisive policy-making. However, climate science studies commonly require addressing complex interoperability issues between data, software, and experimental approaches from multiple fields. Scientific workflow systems provide unparalleled advantages to address these issues, including reproducibility of experiments, provenance capture, software reusability and knowledge sharing. In this paper, we introduce a novel workflow with a series of connected components to perform spatial data preparation, classification of satellite imagery with machine learning algorithms, and assessment of carbon stored by urban trees. To the best of our knowledge, this is the first study that estimates carbon storage for a region in Africa following the guidelines from the Intergovernmental Panel on Climate Change (IPCC).}, keywords = {computational sustainability, machine learning, Semantics}, url = {https://www.semanticscholar.org/paper/Semantic-Workflows-and-Machine-Learning-for-the-of-Garcia-Garijo/cb059bd7de50122a9a7b5a778e04e21f2c02b2c6}, author = {Juan Manuel Carrillo Garcia and Daniel Garijo and Mark Crowley and Rober Carrillo and Yolanda Gil and Katherine Borda} } @unpublished {ghojogh2019theory, title = {The theory behind overfitting, cross validation, regularization, bagging, and boosting: tutorial}, journal = {arXiv preprint arXiv:1905.12787}, year = {2019}, author = {Ghojogh, Benyamin and Crowley, Mark} } @conference {Bhalla2019, title = {Training Cooperative Agents for Multi-Agent Reinforcement Learning}, booktitle = {Proc. of the 18th International Conference on Autonomous Agents and Multiagent Systems (AAMAS 2019)}, year = {2019}, address = {Montreal, Canada}, abstract = {Deep Learning and back-propagation has been successfully used to perform centralized training with communication protocols among multiple agents in a cooperative environment. In this paper we present techniques for centralized training of Multi-Agent (Deep) Reinforcement Learning (MARL) using the model-free Deep Q-Network as the baseline model and message sharing between agents. We present a novel, scalable, centralized MARL training technique, which separates the message learning module from the policy module. The separation of these modules helps in faster convergence in complex domains like autonomous driving simulators. A second contribution uses the centrally trained model to bootstrap training of distributed, independent, cooperative agent policies for execution and thus addresses the challenges of noise and communication bottlenecks in real-time communication channels. This paper theoretically and empirically compares our centralized training algorithms to current research in the field of MARL. We also present and release a new OpenAI-Gym environment which can be used for multi-agent research as it simulates multiple autonomous cars driving cooperatively on a highway.}, keywords = {Autonomous Driving, MARL, Multi-Agent Reinforcement Learning, MultiAgent Systems, reinforcement learning}, author = {Bhalla, Sushrut and Subramanian, Sriram Ganapathi and Crowley, Mark}, editor = {Agmon, N. and Taylor, M.E. and Elkind, E. and Veloso, M.} } @unpublished {ghojogh2019unsupervised, title = {Unsupervised and supervised principal component analysis: Tutorial}, journal = {arXiv preprint arXiv:1906.03148}, year = {2019}, author = {Ghojogh, Benyamin and Crowley, Mark} } @article {Eaton:2018:BSI:3187010.3175509, title = {Blue Sky Ideas in Artificial Intelligence Education from the EAAI 2017 New and Future AI Educator Program}, journal = {AI Matters}, volume = {3}, number = {4}, year = {2018}, month = {feb}, pages = {23{\textendash}31}, publisher = {ACM}, address = {New York, NY, USA}, abstract = {

The 7th Symposium on Educational Advances in Artificial Intelligence (EAAI{\textquoteright}17, co-chaired by Sven Koenig and Eric Eaton) launched the EAAI New and Future AI Educator Program to support the training of early-career university faculty, secondary school faculty, and future educators (PhD candidates or postdocs who intend a career in academia).

As part of the program, awardees were asked to address one of the following {\textquotedblleft}blue sky{\textquotedblright} questions:

  1. How could/should Artificial Intelligence (AI) courses incorporate ethics into the curriculum?
  2. How could we teach AI topics at an early undergraduate or a secondary school level?
  3. AI has the potential for broad impact to numerous disciplines. How could we make AI education more interdisciplinary, specifically to benefit non-engineering fields.

This paper is a collection of their responses, intended to help motivate discussion around these issues in AI education.

}, issn = {2372-3483}, doi = {10.1145/3175502.3175509}, url = {http://doi.acm.org/10.1145/3175502.3175509}, author = {Eaton, Eric and Koenig, Sven and Schulz, Claudia and Maurelli, Francesco and Lee, John and Eckroth, Joshua and Crowley, Mark and Freedman, Richard G and Cardona-Rivera, Rogelio E and Machado, Tiago and Williams, Tom} } @conference {Subramanian2018CCAI, title = {Combining MCTS and A3C for prediction of spatially spreading processes in forest wildfire settings}, booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)}, volume = {10832 LNAI}, year = {2018}, pages = {285{\textendash}291}, address = {Toronto, Ontario, Canada}, abstract = {In recent years, Deep Reinforcement Learning (RL) algorithms have shown super-human performance in a variety Atari and classic board games like chess and GO. Research into applications of RL in other domains with spatial considerations like environmental planning are still in their nascent stages. In this paper, we introduce a novel combination of Monte-Carlo Tree Search (MCTS) and A3C algorithms on an online simulator of a wildfire, on a pair of forest fires in Northern Alberta (Fort McMurray and Richardson fires) and on historical Saskatchewan fires previously compared by others to a physics-based simulator. We conduct several experiments to predict fire spread for several days before and after the given spatial information of fire spread and ignition points. Our results show that the advancements in Deep RL applications in the gaming world have advantages in spatially spreading real-world problems like forest fires. {\textcopyright} Springer International Publishing AG, part of Springer Nature 2018.}, keywords = {a3c, computational sustainability, monte-carlo tree search, reinforcement-learning, Wildfire-Management}, isbn = {9783319896557}, issn = {16113349}, doi = {10.1007/978-3-319-89656-4_28}, author = {Ganapathi Subramanian, Sriram and Crowley, Mark} } @conference {Subramanian2018, title = {Combining MCTS and A3C for Prediction of Spatially Spreading Processes in Forest Wildfire Settings}, booktitle = {Canadian Conference on Artificial Intelligence}, volume = {10832}, year = {2018}, pages = {285-291}, publisher = {Springer}, organization = {Springer}, edition = {Lecture Notes in Artificial Intelligence}, address = {Toronto, Ontario, Canada}, abstract = {

\ In recent years, Deep Reinforcement Learning (RL) algorithms have shown super-human performance in a variety Atari and classic board games like chess and GO. Research into applications of RL in other domains with spatial considerations like environmental planning are still in their nascent stages. In this paper, we introduce a novel combination of Monte-Carlo Tree Search (MCTS) and A3C algorithms on an online simulator of a wildfire, on a pair of forest fires in Northern Alberta (Fort McMurray and Richardson fires) and on historical Saskatchewan fires previously compared by others to a physics-based simulator. We conduct several experiments to predict fire spread for several days before and after the given spatial information of fire spread and ignition points. Our results show that the advancements in Deep RL applications in the gaming world have advantages in spatially spreading real-world problems like forest fires.\ 

}, keywords = {a3c, computational sustainability, fire, forest wild-, monte-carlo tree search, reinforcement learning}, url = {https://link.springer.com/chapter/10.1007/978-3-319-89656-4_28}, author = {Subramanian, Sriram Ganapathi and Crowley, Mark} } @conference {Subramanian2018a, title = {Decision Assist For Self-Driving Cars}, booktitle = {31st Canadian Conference on Artificial Intelligence, Candian AI 2018}, volume = {10832}, year = {2018}, pages = {381-387}, publisher = {Springer}, organization = {Springer}, edition = {Lecture Notes in Artificial Intelligence}, address = {Toronto, Ontario, Canada}, abstract = {Research into self-driving cars has grown enormously in the last decade primarily due to the advances in the fields of machine intelligence and image processing. An under-appreciated aspect of self-driving cars is actively avoiding high traffic zones, low visibility zones, and routes with rough weather conditions by learning different conditions and making decisions based on trained experiences.\ 
This paper addresses this challenge by introducing a novel hierarchical structure for dynamic path planning and experiential learning for vehicles.\ 
A multistage system is proposed for detecting and compensating for weather, lighting, and traffic conditions as well as a novel adaptive path planning algorithm named \textbf{Checked State A3C}. This algorithm improves upon the existing A3C Reinforcement Learning (RL) algorithm by adding state memory which provides the ability to learn an adaptive model of the best decisions to take from experience.}, keywords = {autonomous cars, forcement learning, machine learning, obstacle avoidance, path planning, rein-, weather estimation}, url = {https://link.springer.com/chapter/10.1007\%2F978-3-319-89656-4_44}, author = {Subramanian, Sriram Ganapathi and Sambee, Jaspreet Singh and Ghojogh, Benyamin and Crowley, Mark} } @conference {14507, title = {Principal Sample Analysis for Data Reduction}, booktitle = {International Conference on Big Knowledge (ICBK) }, year = {2018}, publisher = {IEEE, 2018.}, organization = {IEEE, 2018.}, address = {Singapore}, abstract = {Data reduction is an essential technique used for purifying data, training discriminative models more efficiently, encouraging generalizability, and for using less storage space for memory-limited systems.\ 
The literature on data reduction focuses mostly on dimensionality reduction, however, data sample reduction (i.e. removal of data points from a dataset) has its own benefits and is no less important given growing sizes of datasets and the growing need for usable data analysis methods on the network edge.
This paper proposes a new data sample reduction method, Principal Sample Analysis (PSA), which reduces the number (population) of data samples as a preprocessing step for classification. PSA ranks the samples of each class considering how well they represent it and enables better discriminative learning by using the sparsity and similarity of samples at the same time. Data sample reduction then occurs by cutting off the lowest ranked samples. The PSA method can work alongside any other data reduction/expansion and classification method. Experiments are carried out on three datasets (WDBC, AT\&T, and MNIST) with contrasting characteristics and show the state-of-the-art effectiveness of the proposed method.\ }, author = {Benyamin Ghojogh and Mark Crowley} } @article {10365, title = {Using Spatial Reinforcement Learning to Build Forest Wildfire Dynamics Models from Satellite Images}, journal = {Frontiers in ICT: Environmental Informatics}, year = {2018}, abstract = {Machine learning algorithms have increased tremendously in power in recent years but have yet to be fully utilized in many ecology and sustainable resource management domains such as wildlife reserve design, forest fire management and invasive species spread. One thing these domains have in common is that they contain dynamics that can be characterized as a Spatially Spreading Process (SSP) which requires many parameters to be set precisely to model the dynamics, spread rates and directional biases of the elements which are spreading.\ We present related work in Artificial Intelligence and Machine Learning for SSP sustainability domains including forest wildfire prediction.\ We then introduce a novel approach for learning in SSP domains using Reinforcement Learning (RL) where fire is the agent at any cell in the landscape and the set of actions the fire can take from a location at any point in time includes spreading North, South, East, West or not spreading.\ This approach inverts the usual RL setup since the dynamics of the corresponding Markov Decision Process (MDP) is a known function for immediate wildfire spread.\ Meanwhile, we learn an agent policy for a predictive model of the dynamics of a complex spatially-spreading process.\ Rewards are provided for correctly classifying which cells are on fire or not compared to satellite and other related data.\ We examine the behaviour of five RL algorithms on this problem: Value Iteration, Policy Iteration, Q-Learning, Monte Carlo Tree Search and Asynchronous Advantage Actor-Critic (A3C).\ We compare to a Gaussian process based supervised learning approach and discuss the relation of our approach to manually constructed, state-of-the-art methods from forest wildfire modelling.\ We also discuss the relation of our approach to manually constructed, state-of-the-art methods from forest wildfire modelling.\ We validate our approach with satellite image data of two massive wildfire events in Northern Alberta, Canada; the Fort McMurray fire of 2016 and the Richardson fire of 2011. The results show that we can learn predictive, agent-based policies as models of spatial dynamics using RL on readily available satellite images that other methods and have many additional advantages in terms of generalizability and interpretability.}, url = {https://www.frontiersin.org/articles/10.3389/fict.2018.00006/abstract}, author = {Subramanian, Sriram Ganapathi and Crowley, Mark} } @conference {Maryam2017, title = {Application of Probabilistically-Weighted Graphs to Image-Based Diagnosis of Alzheimer\&$\#$39;s Disease using Diffusion MRI.}, booktitle = {SPIE Medical Imaging Conference on Computer-Aided Diagnosis}, volume = {10134}, year = {2017}, month = {March 3}, publisher = {International Society for Optics and Photonics}, organization = {International Society for Optics and Photonics}, address = {Orlando, FL, United States}, abstract = {

The world{\textquoteright}s aging population has given rise to an increasing awareness towards neurodegenerative disorders, including Alzheimers Disease (AD). Treatment options for AD are currently limited, but it is believed that future success depends on our ability to detect the onset of the disease in its early stages. The most frequently used tools for this include neuropsychological assessments, along with genetic, proteomic, and image-based diagnosis. Recently, the applicability of Diffusion Magnetic Resonance Imaging (dMRI) analysis for early diagnosis of AD has also been reported. The sensitivity of dMRI to the microstructural organization of cerebral tissue makes it particularly well-suited to detecting changes which are known to occur in the early stages of AD. Existing dMRI approaches can be divided into two broad categories: region-based and tract-based. In this work, we propose a new approach, which extends region-based approaches to the simultaneous characterization of multiple brain regions. Given a predefined set of features derived from dMRI data, we compute the probabilistic distances between different brain regions and treat the resulting connectivity pattern as an undirected, fully-connected graph. The characteristics of this graph are then used as markers to discriminate between AD subjects and normal controls (NC). Although in this preliminary work we omit subjects in the prodromal stage of AD, mild cognitive impairment (MCI), our method demonstrates perfect separability between AD and NC subject groups with substantial margin, and thus holds promise for fine-grained stratification of NC, MCI and AD populations. {\textcopyright} (2017) COPYRIGHT Society of Photo-Optical Instrumentation Engineers (SPIE). Downloading of the abstract is permitted for personal use only.

}, url = {http://dx.doi.org/10.1117/12.2254164}, author = {Maryam, Syeda and McCrackin, Laura and Crowley, Mark and Rathi, Yogesh and Michailovich, Oleg} } @conference {Subramanian2017, title = {Learning Forest Wildfire Dynamics from Satellite Images Using Reinforcement Learning}, booktitle = {Conference on Reinforcement Learning and Decision Making}, year = {2017}, address = {Ann Arbor, MI, USA.}, keywords = {asynchronous advantage actor-critic, deep reinforcement learning, forest wildfires, images, satellite, spatial informa-, spatiotemporal model, tion, value iteration}, author = {Ganapathi Subramanian, Sriram and Crowley, Mark} } @conference {salem-ecrts-2016, title = {Anomaly Detection Using Inter-Arrival Curves for Real-time Systems}, booktitle = {2016 28th Euromicro Conference on Real-Time Systems}, year = {2016}, month = {jul}, pages = {97{\textendash}106}, address = {Toulouse, France}, abstract = {{\textendash}-Real-time embedded systems are a significant class of applications, poised to grow even further as automated vehicles and the Internet of Things become a reality. An important problem for these systems is to detect anomalies during operation. Anomaly detection is a form of classification, which can be driven by data collected from the system at execution time. We propose inter-arrival curves as a novel analytic modelling technique for discrete event traces. Our approach relates to the existing technique of arrival curves and expands the technique to anomaly detection. Inter-arrival curves analyze the behaviour of events within a trace by providing upper and lower bounds to their inter-arrival occurrence. We exploit inter-arrival curves in a classification framework that detects deviations within these bounds for anomaly detection. Also, we show how inter-arrival curves act as good features to extract recurrent behaviour that these systems often exhibit. We demonstrate the feasibility and viability of the fully implemented approach with an industrial automotive case study (CAN traces) as well as a deployed aerospace case study (RTOS kernel traces).}, keywords = {anomaly detection, arrival curves, embedded systems, proj-MassiveTraceAnomalyDetection, trace analysis}, isbn = {978-1-5090-2811-5}, issn = {10683070}, doi = {10.1109/ECRTS.2016.22}, author = {Salem, Mahmoud and Crowley, Mark and Fischmeister, Sebastian} } @conference {Salem2016, title = {Inter-Arrival Curves for Multi-Mode and Online Anomaly Detection}, booktitle = {Euromicro Conference on Real-Time Systems 2016 - Work-in-Progress Proceedings}, year = {2016}, address = {Toulouse, France}, keywords = {anomaly detection, proj-MassiveTraceAnomalyDetection}, author = {Salem, Mahmoud and Crowley, Mark and Fischmeister, Sebastian} } @conference {Salem2016, title = {Inter-Arrival Curves for Multi-Mode and Online Anomaly Detection}, booktitle = {Euromicro Conference on Real-Time Systems 2016 - Work-in-Progress Proceedings}, year = {2016}, address = {Toulouse, France}, keywords = {anomaly detection, proj-MassiveTraceAnomalyDetection}, author = {Salem, Mahmoud and Crowley, Mark and Fischmeister, Sebastian} } @conference {crowley2015answering, title = {Answering Simple Questions About Spatially Spreading Systems}, booktitle = {2015 Summer Solstice: 7th International Conference on Discrete Models of Complex Systems}, year = {2015}, author = {Crowley, Mark} } @article {Taleghan2015, title = {PAC Optimal MDP Planning with Application to Invasive Species Management}, journal = {Journal of Machine Learning Research}, volume = {16}, year = {2015}, pages = {3877{\textendash}3903}, abstract = {In a simulator-defined MDP, the Markovian dynamics and rewards are provided in the form of a simulator from which samples can be drawn. This paper studies MDP planning algorithms that attempt to minimize the number of simulator calls before terminating and outputting a policy that is approximately optimal with high probability. The paper introduces two heuristics for efficient exploration and an improved confidence interval that enables earlier termination with probabilis- tic guarantees. We prove that the heuristics and the confidence interval are sound and produce with high probability an approximately optimal policy in polynomial time. Experiments on two benchmark problems and two instances of an invasive species management problem show that the improved confidence intervals and the new search heuristics yield reductions of between 8\% and 47\% in the number of simulator calls required to reach near-optimal policies.}, keywords = {computational sustainability, Good- Turing estimate, grant-wici16, invasive species management, machine learning, Markov decision processes, mdp, optimization, reinforcement-learning}, issn = {15337928}, url = {http://jmlr.org/papers/v16/taleghan15a.html}, author = {Taleghan, Majid Alkaee and Dietterich, Thomas G. and Crowley, Mark and Hall, Kim and Albers, H. Jo} } @article {crowleyieee2013, title = {Using equilibrium policy gradients for spatiotemporal planning in forest ecosystem management}, journal = {IEEE Transactions on Computers}, volume = {63}, year = {2014}, pages = {142{\textendash}154}, publisher = {IEEE computer Society Digital Library. IEEE Computer Society.}, abstract = {

Spatiotemporal planning involves making choices at multiple locations in space over some planning horizon to maximize utility and satisfy various constraints. In Forest Ecosystem Management, the problem is to choose actions for thousands of locations each year including harvesting, treating trees for fire or pests, or doing nothing. The utility models could place value on sale of lumber, ecosystem sustainability or employment levels and incorporate legal and logistical constraints on actions such as avoiding large contiguous areas of clearcutting. Simulators developed by forestry researchers provide detailed dynamics but are generally inaccesible black boxes. We model spatiotemporal planning as a factored Markov decision process and present a policy gradient planning algorithm to optimize a stochastic spatial policy using simulated dynamics. It is common in environmental and resource planning to have actions at different locations be spatially interelated; this makes representation and planning challenging. We define a global spatial policy in terms of interacting local policies defining distributions over actions at each location conditioned on actions at nearby locations. Markov chain Monte Carlo simulation is used to sample landscape policies and estimate their gradients. Evaluation is carried out on a forestry planning problem with 1,880 locations using a variety of value models and constraints. Index

}, keywords = {computational sustainability, Ecosystem management, Forestry planning, grant-wici16, machine learning, Markov decision processes, optimization, Policy gradient planning, proj-spatiallyspreadingprocess, reinforcement learning, spatiotemporal planning}, issn = {00189340}, doi = {10.1109/TC.2013.113}, url = {http://doi.ieeecomputersociety.org/10.1109/TC.2013.113}, author = {Crowley, Mark} } @article {houtman2013, title = {Allowing a wildfire to burn: Estimating the effect on future fire suppression costs}, journal = {International Journal of Wildland Fire}, volume = {22}, year = {2013}, pages = {871{\textendash}882}, abstract = {Where a legacy of aggressive wildland fire suppression has left forests in need of fuel reduction, allowing wildland fire to burn may provide fuel treatment benefits, thereby reducing suppression costs from subsequent fires. The least-cost-plus-net-value-change model of wildland fire economics includes benefits of wildfire in a framework for evaluating suppression options. In this study, we estimated one component of that benefit {\textendash} the expected present value of the reduction in suppression costs for subsequent fires arising from the fuel treatment effect of a current fire. To that end, we employed Monte Carlo methods to generate a set of scenarios for subsequent fire ignition and weather events, which are referred to as sample paths, for a study area in central Oregon. We simulated fire on the landscape over a 100-year time horizon using existing models of fire behaviour, vegetation and fuels development, and suppression effectiveness, and we estimated suppression costs using an existing suppression cost model. Our estimates suggest that the potential cost savings may be substantial. Further research is needed to estimate the full least-cost-plus-net-value-change model. This line of research will extend the set of tools available for developing wildfire management plans for forested landscapes.}, keywords = {bio-economic modelling, forest economics, forest fire policy, grant-wici16, proj-spatiallyspreadingprocess, spatial simulation, wildland fire management}, issn = {10498001}, doi = {10.1071/WF12157}, author = {Houtman, Rachel M. and Montgomery, Claire A. and Gagnon, Aaron R. and Calkin, David E. and Dietterich, Thomas G. and McGregor, Sean and Crowley, Mark} } @conference {PooleCrowley2013, title = {Cyclic causal models with discrete variables: Markov chain equilibrium semantics and sample ordering}, booktitle = {IJCAI International Joint Conference on Artificial Intelligence}, year = {2013}, pages = {1060{\textendash}1068}, address = {Beijing, China}, abstract = {We analyze the foundations of cyclic causal models for discrete variables, and compare structural equation models (SEMs) to an alternative semantics as the equilibrium (stationary) distribution of a Markov chain. We show under general conditions, discrete cyclic SEMs cannot have independent noise; even in the simplest case, cyclic structural equation models imply constraints on the noise. We give a formalization of an alternative Markov chain equilibrium semantics which requires not only the causal graph, but also a sample order. We show how the resulting equilibrium is a function of the sample ordering, both theoretically and empirically.}, keywords = {cyclic causality, probabilistic inference}, isbn = {9781577356332}, issn = {10450823}, url = {http://dl.acm.org/citation.cfm?id=2540281}, author = {Poole, David and Crowley, Mark} } @conference {PooleCrowley2013, title = {Cyclic causal models with discrete variables: Markov chain equilibrium semantics and sample ordering}, booktitle = {IJCAI International Joint Conference on Artificial Intelligence}, year = {2013}, pages = {1060{\textendash}1068}, address = {Beijing, China}, abstract = {We analyze the foundations of cyclic causal models for discrete variables, and compare structural equation models (SEMs) to an alternative semantics as the equilibrium (stationary) distribution of a Markov chain. We show under general conditions, discrete cyclic SEMs cannot have independent noise; even in the simplest case, cyclic structural equation models imply constraints on the noise. We give a formalization of an alternative Markov chain equilibrium semantics which requires not only the causal graph, but also a sample order. We show how the resulting equilibrium is a function of the sample ordering, both theoretically and empirically.}, keywords = {Causality, probabilistic inference}, isbn = {9781577356332}, issn = {10450823}, url = {http://dl.acm.org/citation.cfm?id=2540281}, author = {Poole, David and Crowley, Mark} } @conference {Dietterich2013, title = {PAC Optimal Planning for Invasive Species Management: Improved Exploration for Reinforcement Learning from Simulator-Defined MDPs}, booktitle = {Proceedings of the AAAI Conference on Artificial Intelligence (AAAI-2013)}, year = {2013}, pages = {7}, address = {Bellevue, WA, USA}, abstract = {Often the most practical way to define a Markov Decision Process (MDP) is as a simulator that, given a state and an action, produces a resulting state and immediate reward sam- pled from the corresponding distributions. Simulators in nat- ural resource management can be very expensive to execute, so that the time required to solve such MDPs is dominated by the number of calls to the simulator. This paper presents an algorithm, DDV, that combines improved confidence inter- vals on the Q values (as in interval estimation) with a novel upper bound on the discounted state occupancy probabilities to intelligently choose state-action pairs to explore. We prove that this algorithm terminates with a policy whose value is within $ε$ of the optimal policy (with probability 1-$δ$) after making only polynomially-many calls to the simulator. Ex- periments on one benchmark MDP and on an MDP for inva- sive species management show very large reductions in the number of simulator calls required.}, keywords = {computational sustainability, invasive species, machine learning, mdp, PAC learning, planning}, isbn = {9781577356158}, url = {http://www.aaai.org/ocs/index.php/AAAI/AAAI13/paper/view/6478}, author = {Dietterich, Thomas G and Taleghan, Majid Alkaee and Crowley, Mark} } @conference {Crowley2013, title = {Policy Gradient Optimization Using Equilibrium Policies for Spatial Planning Domains}, booktitle = {13th INFORMS Computing Society Conference}, year = {2013}, address = {Santa Fe, NM, United States}, author = {Crowley, Mark} } @conference {Hall2012, title = {Managing Invasive Species in a River Network}, booktitle = {Third International Conference on Computational Sustainability}, year = {2012}, address = {Copenhagen, Denmark}, abstract = {There are many pest species that invade valuable ecosystems and damage ecosystem services. An important management goal is to limit the spread of these species or even eradicate them. From the computational perspective, these problems can be formulated as Markov Decision Processes. However, solving them is difficult, because of their spatial nature. Viewed abstractly, the current state of the ecosystem can be formalized as a graph. Each node of the graph may be invaded or not, and at each time step, a {\textquotedblleft}local{\textquotedblright} action must be selected at each node. Consequently, at the level of the whole MDP, each global action is a vector of local actions of length N, the number of nodes in the graph. The exponentially large action space raises major computational problems. During the state transitions, the invasive species can spread along the edges of the graph. We study a specific instance of this general problem based on the spread of Tamarisk in river networks. In this formulation, we work with the dual of the standard river network graph. Each node corresponds to a stretch of the river between two confluences (also called a {\textquotedblleft}reach{\textquotedblright}). We divide each reach into H {\textquotedblleft}slots{\textquotedblright}, where each slot can either be occupied by a single native tree, occupied by a single Tamarisk tree, or empty. In each time step, the plants (both native and invader) first must survive through winter (with some probability). Then, the surviving trees produce seeds that disperse through the river network according to a dispersal kernel that prefers downstream movement with high probability. Each dispersing seed lands at some slot in some node of the network. All seeds arriving at a slot compete to establish themselves in that slot (if the slot is empty). In our model, there are three local actions: simple eradication of invasive plants, eradication followed by planting of native plants, or doing nothing at all. Doing nothing costs nothing, while eradication is more expensive, and eradication plus planting is the most expensive. One local action must be chosen for each node (reach) at each time step, and it applies to all H slots at that node. The actions only succeed stochastically. The planning goal is to minimize the expected discounted sum of the cost of invasive damage and the cost of management actions. To solve this planning problem, we apply the standard value iteration algorithm. However, standard value iteration requires a fully-specified state transition matrix. In our problem, it is easy to write a simulator for drawing samples of the dispersal and competition processes, but it is computationally intractable to compute the exact transition probabilities. Hence, we estimate those probabilities by drawing a large number of samples from the simulator. Once we have an optimal policy on the approximated MDP, we explore its behaviour both directly and via simulated rollouts. The optimal policy can be visualized with one picture per state annotated by the types of plants in each reach and the optimal action that should be taken in each reach. We also run the optimal policy forward in time on many simulated trajectories in order to gather statistics about its behaviour such as: how long it takes to reach a steady state; average cost of a policy over time; or the frequency with which completely invaded or uninvaded states are reached. Comparisons of these statistics show that in many situations our optimal policies greatly outperform established rules of thumb from the literature. An example of a rule of thumb is to always prioritize treatment of upstream over downstream reaches. Another rule targets the most invaded reaches (those with the most slots occupied by Tamarisk) first. We have found that the optimality of these rules of thumb depends strongly on the dispersal and competition parameters. Some contributions this work makes to bioeconomics and ecology are showing evidence that the stochastic spread of the invasion process needs to be modelled and that the spatial characteristics of the system under invasion are relevant to optimal management. The computational challenges raised by this problem have led us into research on methods for planning that can provide bounds on the optimality of the policy while minimizing the number of calls to expensive simulators. We are also studying algorithms that can scale to large graphs with thousands of nodes and large out-degrees. Finally, we see potential in these problems for intelligently reducing the size of the state space and the complexity of the policy description by utilizing domain knowledge.}, keywords = {computational sustainability, ecology, invasive species, reinforcement learning, value iteration}, url = {http://www.cs.ubc.ca/~crowley/papers/compsust2012.pdf}, author = {Hall, Kim and Taleghan, Majid Alkaee and Albers, Heidi J. and Dietterich, Thomas and Crowley, Mark} } @mastersthesis {Crowley2011thesis, title = {Equilibrium Policy Gradients for Spatiotemporal Planning}, year = {2011}, pages = {187 pages}, school = {University of British Columbia}, type = {phdPh.D.}, abstract = {In spatiotemporal planning, agents choose actions at multiple locations in space over some planning horizon to maximize their utility and satisfy various constraints. In forestry planning, for example, the problem is to choose actions for thousands of locations in the forest each year. The actions at each location could include harvesting trees, treating trees against disease and pests, or doing nothing. A utility model could place value on sale of forest products, ecosystem sustainability or employment levels, and could incorporate legal and logistical constraints such as avoiding large contiguous areas of clearcutting and managing road access. Planning requires a model of the dynamics. Existing simulators developed by forestry researchers can provide detailed models of the dynamics of a forest over time, but these simulators are often not designed for use in automated planning. This thesis presents spatiotemoral planning in terms of factored Markov decision processes. A policy gradient planning algorithm optimizes a stochastic spatial policy using existing simulators for dynamics. When a planning problem includes spatial interaction between locations, deciding on an action to carry out at one location requires considering the actions performed at other locations. This spatial interdependence is common in forestry and other environmental planning problems and makes policy representation and planning challenging. We define a spatial policy in terms of local policies defined as distributions over actions at one location conditioned upon actions at other locations. A policy gradient planning algorithm using this spatial policy is presented which uses Markov Chain Monte Carlo simulation to sample the landscape policy, estimate its gradient and use this gradient to guide policy improvement. Evaluation is carried out on a forestry planning problem with 1880 locations using a variety of value models and constraints. The distribution over joint actions at all locations can be seen as the equilibrium of a cyclic causal model. This equilibrium semantics is compared to Structural Equation Models. We also define an algorithm for approximating the equilibrium distribution for cyclic causal networks which exploits graphical structure and analyse when the algorithm is exact.}, keywords = {computational sustainability, graphical models, machine learning, mdp, reinforcement learning, spatial planning, spatiotemporal planning}, url = {http://hdl.handle.net/2429/38971}, author = {Crowley, Mark} } @conference {Crowley2011, title = {Policy gradient planning for environmental decision making with existing simulators}, booktitle = {25th AAAI Conference on Artificial Intelligence (AAAI-11)}, volume = {2}, year = {2011}, pages = {1323{\textendash}1330}, address = {San Francisco}, abstract = {In environmental and natural resource planning domains actions are taken at a large number of locations over multiple time periods. These problems have enormous state and action spaces, spatial correlation between actions, uncertainty and complex utility models. We present an approach for modeling these planning problems as factored Markov decision processes. The reward model can contain local and global components as well as spatial constraints between locations. The transition dynamics can be provided by existing simulators developed by domain experts. We propose a landscape policy defined as the equilibrium distribution of a Markov chain built from many locally-parameterized policies. This policy is optimized using a policy gradient algorithm. Experiments using a forestry simulator demonstrate the algorithm{\textquoteright}s ability to devise policies for sustainable harvest planning of a forest. Copyright {\textcopyright} 2011, Association for the Advancement of Artificial Intelligence. All rights reserved.}, keywords = {mdp, reinforcement learning, spatiotemporal planning, thesis research}, isbn = {9781577355090}, url = {https://www.scopus.com/record/display.uri?eid=2-s2.0-80055051332\&origin=inward\&txGid=de2006c39235aac9ba20cf0e76073dd9}, author = {Crowley, Mark and Poole, David} } @conference {Patitsas2010, title = {Circuits and logic in the lab : Toward a coherent picture of computation}, booktitle = {15th Western Canadian Conference on Computing Education}, year = {2010}, address = {Kelowna, BC, Canada}, abstract = {We describe extensive modifications made over time to a first year computer science course at the University of British Columbia covering logic and digital circuits (among other topics). Smoothly integrating the hardware-based labs with the more theory-based lectures into a cohesive picture of computation has always been a challenge in this course. The seeming disconnect between implementation and abstraction has historically led to frustration and dissatisfaction among students. We describe changes to the lab curriculum, equipment logistics, the style of in-lab activities and evaluation. We have also made logistical changes to the management and ongoing training of teaching assistants, allowing us to better anchor our larger course story into the lab curriculum. These changes have greatly improved student and TA opinions of the lab experience, as well as the overall course.}, keywords = {curriculum, education research, Pedagogy, survey}, isbn = {9781450300988}, doi = {10.1145/1806512.1806523}, url = {http://www.cs.ubc.ca/~crowley/papers/wccce2010.pdf}, author = {Patitsas, Elizabeth and Voll, Kimberly and Crowley, Mark and Wolfman, Steven} } @conference {Crowley2009, title = {Seeing the Forest Despite the Trees : Large Scale Spatial-Temporal Decision Making}, booktitle = {Conference on Uncertainty in Artificial Intelligence (UAI09)}, year = {2009}, pages = {126{\textendash}134}, address = {Montreal, Canada}, abstract = {We introduce a challenging real world planning problem where actions must be taken at each location in a spatial area at each point in time. We use forestry planning as the motivating application. In Large Scale Spatial-Temporal (LSST) planning problems, the state and action spaces are defined as the cross-products of many local state and action spaces spread over a large spatial area such as a city or forest. These problems possess state uncertainty, have complex utility functions involving spatial constraints and we generally must rely on simulations rather than an explicit transition model. We define LSST problems as reinforcement learning prob- lems and present a solution using policy gradients. We compare two different policy formulations: an explicit policy that identifies each location in space and the action to take there; and an abstract policy that defines the proportion of actions to take across all locations in space. We show that the abstract policy is more robust and achieves higher rewards with far fewer parameters than the elementary policy. This abstract policy is also a better fit to the properties that practitioners in LSST problem domains require for such methods to be widely useful}, keywords = {computational sustainability, Forest Management, inference, mdp, planning, spatiotemporal planning}, url = {http://www.cs.ubc.ca/ crowley/papers/uai09-mark-crowley.pdf}, author = {Crowley, Mark and Nelson, John and Poole, David} } @conference {Crowley2009, title = {Seeing the Forest Despite the Trees : Large Scale Spatial-Temporal Decision Making}, booktitle = {Conference on Uncertainty in Artificial Intelligence (UAI09)}, year = {2009}, pages = {126{\textendash}134}, address = {Montreal, Canada}, abstract = {We introduce a challenging real world planning problem where actions must be taken at each location in a spatial area at each point in time. We use forestry planning as the motivating application. In Large Scale Spatial-Temporal (LSST) planning problems, the state and action spaces are defined as the cross-products of many local state and action spaces spread over a large spatial area such as a city or forest. These problems possess state uncertainty, have complex utility functions involving spatial constraints and we generally must rely on simulations rather than an explicit transition model. We define LSST problems as reinforcement learning prob- lems and present a solution using policy gradients. We compare two different policy formulations: an explicit policy that identifies each location in space and the action to take there; and an abstract policy that defines the proportion of actions to take across all locations in space. We show that the abstract policy is more robust and achieves higher rewards with far fewer parameters than the elementary policy. This abstract policy is also a better fit to the properties that practitioners in LSST problem domains require for such methods to be widely useful}, keywords = {computational sustainability, forestry, inference, mdp, planning, spatiotemporal planning}, url = {http://www.cs.ubc.ca/~crowley/papers/uai09-mark-crowley.pdf}, author = {Crowley, Mark and Nelson, John and Poole, David} } @conference {Crowley2007, title = {Adding Local Constraints to Bayesian Networks}, booktitle = {Advances in Artificial Intelligence}, series = {Lecture Notes in Computer Science}, volume = {4509}, year = {2007}, pages = {344{\textendash}355}, publisher = {Springer Berlin Heidelberg}, organization = {Springer Berlin Heidelberg}, edition = {4509}, address = {Canadian AI Conference, Montreal, Quebec, Canada, 2007.}, abstract = {When using Bayesian networks, practitioners often express constraints among variables by conditioning a common child node to induce the desired distribution. For example, an {\textquoteleft}or{\textquoteright} constraint can be easily expressed by a node modelling a logical {\textquoteleft}or{\textquoteright} of its parents{\textquoteright} values being conditioned to true. This has the desired effect that at least one parent must be true. However, conditioning also alters the distributions of further ancestors in the network. In this paper we argue that these side effects are undesirable when constraints are added during model design. We describe a method called shielding to remove these side effects while remaining within the directed language of Bayesian networks. This method is then compared to chain graphs which allow undirected and directed edges and which model equivalent distributions. Thus, in addition to solving this common modelling problem, shielded Bayesian networks provide a novel method for implementing chain graphs with existing Bayesian network tools.}, keywords = {bayesian networks, Computer Science, constraints, probabilistic inference}, isbn = {978-3-540-72664-7}, issn = {0302-9743}, doi = {10.1007/978-3-540-72665-4}, url = {http://www.springerlink.com/content/u1j205nhr750m717/}, author = {Crowley, Mark and Boerlage, Brent and Poole, David and Kobti, Ziad and Wu, Dan}, editor = {Kobti, Ziad and Wu, Dan} } @mastersthesis {Crowley2005, title = {Shielding Against Conditioning Side-Effects in Graphical Models}, year = {2005}, school = {University of British Columbia}, type = {PhD Thesis}, keywords = {bayesian networks, constraints, inference, influence diagrams, side-effects}, author = {Crowley, Mark} } @unpublished {Crowley2004, title = {Evaluating Influence Diagrams}, journal = {Unpublished Working Paper}, year = {2004}, abstract = {In this unpublished literature review we will survey the history of Influence Diagrams from their origin in Decision Theory to modern AI uses. We will compare the various methods that have been used to find an optimal strategy for a given influence diagram. These methods have various advantages under different assumptions and there is a pro- gression to the present of richer, more general solutions. We also look at an abstract algorithm used as an informal tool and determine whether it is equivalent to other formal methods in the literature.}, author = {Crowley, Mark} }