Merge pull request #23 from cassandra-parent/main

Final Project Submission for Cassandra Parent -- RNNs vs Transformers

assets/bibliography/2023-11-09-TransformersAndRNNs.bib

@@ -29,4 +29,119 @@ @misc{qin2023hierarchically
       eprint={2311.04823},
       archivePrefix={arXiv},
       primaryClass={cs.CL}
-}
+}
+@misc{lstmseries,
+  author={Ludwig, Simone A.},
+  booktitle={2019 IEEE International Conference on Fuzzy Systems (FUZZ-IEEE)}, 
+  title={Comparison of Time Series Approaches applied to Greenhouse Gas Analysis: ANFIS, RNN, and LSTM}, 
+  year={2019},
+  volume={},
+  number={},
+  pages={1-6},
+  doi={10.1109/FUZZ-IEEE.2019.8859013}
+}
+@misc{limitation,
+  author={Talai K., Slimane, O., & Kaabouch N.},
+  booktitle={Springer Link}, 
+  title={Deep learning: systematic review, models, challenges, and research directions}, 
+  year={2023},
+  volume={35},
+  doi={10.1007/s00521-023-08957-4}
+}
+@misc{limitationexplain,
+  author={Linardatos P, Papastefanopoulos V, Kotsiantis S.},
+  booktitle={Entropy}, 
+  title={Explainable AI: A Review of Machine Learning Interpretability Methods}, 
+  year={2020},
+  volume={23},
+  doi={10.3390/e23010018}
+}
+@article{Sherstinsky_2020,
+   title={Fundamentals of Recurrent Neural Network (RNN) and Long Short-Term Memory (LSTM) network},
+   volume={404},
+   ISSN={0167-2789},
+   url={http://dx.doi.org/10.1016/j.physd.2019.132306},
+   DOI={10.1016/j.physd.2019.132306},
+   journal={Physica D: Nonlinear Phenomena},
+   publisher={Elsevier BV},
+   author={Sherstinsky, Alex},
+   year={2020},
+   month=mar, pages={132306} }
+@inproceedings{NIPS2017_3f5ee243,
+      author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
+      booktitle = {Advances in Neural Information Processing Systems},
+      editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
+      pages = {},
+      publisher = {Curran Associates, Inc.},
+      title = {Attention is All you Need},
+      url = {https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf},
+      volume = {30},
+      year = {2017}
+}
+@misc{peng2023rwkv,
+      title={RWKV: Reinventing RNNs for the Transformer Era}, 
+      author={Bo Peng and Eric Alcaide and Quentin Anthony and Alon Albalak and Samuel Arcadinho and Stella Biderman and Huanqi Cao and Xin Cheng and Michael Chung and Matteo Grella and Kranthi Kiran GV and Xuzheng He and Haowen Hou and Jiaju Lin and Przemyslaw Kazienko and Jan Kocon and Jiaming Kong and Bartlomiej Koptyra and Hayden Lau and Krishna Sri Ipsit Mantri and Ferdinand Mom and Atsushi Saito and Guangyu Song and Xiangru Tang and Bolun Wang and Johan S. Wind and Stanislaw Wozniak and Ruichong Zhang and Zhenyuan Zhang and Qihang Zhao and Peng Zhou and Qinghua Zhou and Jian Zhu and Rui-Jie Zhu},
+      year={2023},
+      eprint={2305.13048}
+      }
+@misc{keles2022computational,
+      title={On The Computational Complexity of Self-Attention}, 
+      author={Feyza Duman Keles and Pruthuvi Mahesakya Wijewardena and Chinmay Hegde},
+      year={2022},
+      eprint={2209.04881},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@misc{Kanagachidambaresan2021,
+      author="Kanagachidambaresan, G. R.
+      and Ruwali, Adarsha
+      and Banerjee, Debrup
+      and Prakash, Kolla Bhanu",
+      editor="Prakash, Kolla Bhanu
+      and Kanagachidambaresan, G. R.",
+      title="Recurrent Neural Network",
+      bookTitle="Programming with TensorFlow: Solution for Edge Computing Applications",
+      year="2021",
+      publisher="Springer International Publishing",
+      address="Cham",
+      pages="53--61",
+      abstract="Recurrent neural networks (RNN) are very powerful types of neural networks and are the most promising algorithm because they are the only ones with an internal memory (Boca Raton Mhaskar et al. Learning functions: when is deep better than shallow. arXiv:1603.00988, 2016). RNN is the most preferred algorithm for sequential data that includes speech, text, financial data, audio, video, weather, and much more as it can provide a deeper understanding of sequence and its meaning compared to other algorithms (Deep Learning for Computer Vision: Expert techniques to train advanced neural networks using TensorFlow and Keras. [Authors: RajalingappaaShanmugamani]). It is generally used to obtain predictive results in sequential data.",
+      isbn="978-3-030-57077-4",
+      doi="10.1007/978-3-030-57077-4_7",
+      url="https://doi.org/10.1007/978-3-030-57077-4_7"
+}
+@misc{mhaskar2016learning,
+      title={Learning Functions: When Is Deep Better Than Shallow}, 
+      author={Hrushikesh Mhaskar and Qianli Liao and Tomaso Poggio},
+      year={2016},
+      eprint={1603.00988},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@misc{islam2023comprehensive,
+      title={A Comprehensive Survey on Applications of Transformers for Deep Learning Tasks}, 
+      author={Saidul Islam and Hanae Elmekki and Ahmed Elsebai and Jamal Bentahar and Najat Drawel and Gaith Rjoub and Witold Pedrycz},
+      year={2023},
+      eprint={2306.07303},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@ARTICLE{10.3389/fnbot.2023.1157957,
+      AUTHOR={Pettersson, Julius and Falkman, Petter},   
+
+      TITLE={Comparison of LSTM, Transformers, and MLP-mixer neural networks for gaze based human intention prediction},      
+
+      JOURNAL={Frontiers in Neurorobotics},      
+
+      VOLUME={17},           
+
+      YEAR={2023},      
+
+      URL={https://www.frontiersin.org/articles/10.3389/fnbot.2023.1157957},       
+
+      DOI={10.3389/fnbot.2023.1157957},      
+
+      ISSN={1662-5218},   
+
+      ABSTRACT={Collaborative robots have gained popularity in industries, providing flexibility and increased productivity for complex tasks. However, their ability to interact with humans and adapt to their behavior is still limited. Prediction of human movement intentions is one way to improve the robots adaptation. This paper investigates the performance of using Transformers and MLP-Mixer based neural networks to predict the intended human arm movement direction, based on gaze data obtained in a virtual reality environment, and compares the results to using an LSTM network. The comparison will evaluate the networks based on accuracy on several metrics, time ahead of movement completion, and execution time. It is shown in the paper that there exists several network configurations and architectures that achieve comparable accuracy scores. The best performing Transformers encoder presented in this paper achieved an accuracy of 82.74%, for predictions with high certainty, on continuous data and correctly classifies 80.06% of the movements at least once. The movements are, in 99% of the cases, correctly predicted the first time, before the hand reaches the target and more than 19% ahead of movement completion in 75% of the cases. The results shows that there are multiple ways to utilize neural networks to perform gaze based arm movement intention prediction and it is a promising step toward enabling efficient human-robot collaboration.}
+      }