@conference {1485,
	title = {Asynchronous SGD for DNN Training on Shared-Memory Parallel Architectures},
	booktitle = {Workshop on Scalable Deep Learning over Parallel And Distributed Infrastructures (ScaDL 2020)},
	year = {2020},
	month = {2020-05},
	author = {Florent Lopez and Edmond Chow and Stanimire Tomov and Jack Dongarra}
}
@techreport {1465,
	title = {Asynchronous SGD for DNN Training on Shared-Memory Parallel Architectures},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-20-04},
	year = {2020},
	month = {2020-03},
	publisher = {University of Tennessee, Knoxville},
	abstract = {We present a parallel asynchronous Stochastic Gradient Descent algorithm for shared memory architectures. Different from previous asynchronous algorithms, we consider the case where the gradient updates are not particularly sparse. In the context of the MagmaDNN framework, we compare the parallel efficiency of the asynchronous implementation with that of the traditional synchronous implementation. Tests are performed for training deep neural networks on multicore CPUs and GPU devices.},
	keywords = {Asynchronous iterative methods, Deep learning, gpu, multicore CPU, Stochastic Gradient Descent},
	author = {Florent Lopez and Edmond Chow and Stanimire Tomov and Jack Dongarra}
}
@article {,
	title = {How to Build Your Own Deep Neural Network},
	year = {2020},
	month = {2020-07},
	publisher = {PEARC20},
	keywords = {AI, Deep Neural Networks, dense linear algebra, HPC, ML},
	author = {Kwai Wong and Stanimire Tomov and Daniel Nichols and Rocco Febbo and Florent Lopez and Julian Halloy and Xianfeng Ma}
}
@article {,
	title = {Integrating Deep Learning in Domain Science at Exascale (MagmaDNN)},
	year = {2020},
	month = {2020-12},
	publisher = {DOD HPCMP seminar},
	address = {virtual},
	abstract = {We will present some of the current challenges in the design and integration of deep learning AI with traditional HPC simulations. We evaluate existing packages for readiness to run efficiently deep learning models and applications on large scale HPC systems, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and up-coming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated in MagmaDNN, an open source HPC deep learning framework. 
Many deep learning frameworks are targeted towards data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how these can be provided, e.g., as in MagmaDNN, through a deep integration with existing HPC libraries such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced and mixed-precision and asynchronous optimization methods. Finally, we present illustrations and potential solutions on enhancing traditional compute and data intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated on materials science, imaging, and climate applications.},
	author = {Stanimire Tomov and Kwai Wong and Jack Dongarra and Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Junqi Yin}
}
@techreport {,
	title = {Integrating Deep Learning in Domain Sciences at Exascale},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-20-10},
	year = {2020},
	month = {2020-08},
	publisher = {University of Tennessee},
	abstract = {This paper presents some of the current challenges in designing deep learning artificial intelligence (AI) and integrating it with traditional high-performance computing (HPC) simulations. We evaluate existing packages for their ability to run deep learning models and applications on large-scale HPC systems e ciently, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and upcoming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated into MagmaDNN, an open-source HPC deep learning framework. Many deep learning frameworks are targeted at data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how those needs can be provided (e.g., as in MagmaDNN) through a deep integration with existing HPC libraries, such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced- and mixed-precision, as well as asynchronous optimization methods. Finally, we present illustrations and potential solutions for enhancing traditional compute- and data-intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated in materials science, imaging, and climate applications.},
	author = {Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Stanimire Tomov and Kwai Wong and Junqi Yin}
}
@conference {,
	title = {Integrating Deep Learning in Domain Sciences at Exascale},
	booktitle = {2020 Smoky Mountains Computational Sciences and Engineering Conference (SMC 2020)},
	year = {2020},
	month = {2020-08},
	abstract = {This paper presents some of the current challenges in designing deep learning artificial intelligence (AI) and integrating it with traditional high-performance computing (HPC) simulations. We evaluate existing packages for their ability to run deep learning models and applications on large-scale HPC systems e ciently, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and upcoming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated into MagmaDNN, an open-source HPC deep learning framework. Many deep learning frameworks are targeted at data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how those needs can be provided (e.g., as in MagmaDNN) through a deep integration with existing HPC libraries, such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced- and mixed-precision, as well as asynchronous optimization methods. Finally, we present illustrations and potential solutions for enhancing traditional compute- and data-intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated in materials science, imaging, and climate applications.},
	author = {Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Stanimire Tomov and Kwai Wong and Junqi Yin}
}
@techreport {,
	title = {Mixed Precision LU Factorization on GPU Tensor Cores: Reducing Data Movement and Memory Footprint},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-20-13},
	year = {2020},
	month = {2020-09},
	publisher = {University of Tennessee},
	abstract = {Modern GPUs equipped with mixed precision tensor core units present great potential to accelerate dense linear algebra operations such as LU factorization. However, previous works have focused solely on improving speed, neglecting memory consumption. Indeed, state-of-the-art mixed half/single precision LU factorization algorithms all require the matrix to be stored in single precision. This is explained by the fact that simply switching the storage precision from single to half leads to significant loss of accuracy, forfeiting all accuracy benefits from using tensor core technology. In this article, we propose a new factorization algorithm that is able to store the matrix in half precision without incurring any significant loss of accuracy. Our approach is based on a left-looking scheme employing single precision buffers of controlled size and a mixed precision doubly partitioned algorithm exploiting tensor cores in the panel factorizations. Our numerical results show that compared with the state of the art, the proposed approach is of similar accuracy, up to twice faster, and with only half the data movement and memory footprint.},
	keywords = {High Performance Computing, lu factorization, mixed precision algorithms, numerical linear algebra, NVIDIA GPU, rounding error analysis, tensor cores},
	author = {Florent Lopez and Theo Mary}
}
