[{"language":[{"iso":"eng"}],"ddc":["000"],"user_id":"477","department":[{"_id":"75"}],"project":[{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"},{"name":"SFB 901: SFB 901","_id":"1"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"}],"_id":"32811","status":"public","abstract":[{"text":"The decentralized nature of multi-agent systems requires continuous data exchange to achieve global objectives. In such scenarios, Age of Information (AoI) has become an important metric of the freshness of exchanged data due to the error-proneness and delays of communication systems. Communication systems usually possess dependencies: the process describing the success or failure of communication is highly correlated when these attempts are ``close'' in some domain (e.g. in time, frequency, space or code as in wireless communication) and is, in general, non-stationary. To study AoI in such scenarios, we consider an abstract event-based AoI process $\\Delta(n)$, expressing time since the last update: If, at time $n$, a monitoring node receives a status update from a source node (event $A(n-1)$ occurs), then $\\Delta(n)$ is reset to one; otherwise, $\\Delta(n)$ grows linearly in time. This AoI process can thus be viewed as a special random walk with resets. The event process $A(n)$ may be nonstationary and we merely assume that its temporal dependencies decay sufficiently, described by $\\alpha$-mixing. We calculate moment bounds for the resulting AoI process as a function of the mixing rate of $A(n)$. Furthermore, we prove that the AoI process $\\Delta(n)$ is itself $\\alpha$-mixing from which we conclude a strong law of large numbers for $\\Delta(n)$. These results are new, since AoI processes have not been studied so far in this general strongly mixing setting. This opens up future work on renewal processes with non-independent interarrival times.","lang":"eng"}],"type":"conference","publication":"Proceedings of the 58th Allerton Conference on Communication, Control, and Computing","conference":{"name":"58th Allerton Conference on Communication, Control, and Computing"},"title":"Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law","author":[{"orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","full_name":"Redder, Adrian","id":"52265","first_name":"Adrian"},{"orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","id":"66937","full_name":"Ramaswamy, Arunselvan","first_name":"Arunselvan"},{"first_name":"Holger","last_name":"Karl","id":"126","full_name":"Karl, Holger"}],"date_created":"2022-08-15T09:59:17Z","date_updated":"2022-11-18T09:31:19Z","citation":{"ama":"Redder A, Ramaswamy A, Karl H. Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law. In: <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>. ; 2022.","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law,” presented at the 58th Allerton Conference on Communication, Control, and Computing, 2022.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law.” In <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>, 2022.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law. <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>. 58th Allerton Conference on Communication, Control, and Computing.","bibtex":"@inproceedings{Redder_Ramaswamy_Karl_2022, title={Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law}, booktitle={Proceedings of the 58th Allerton Conference on Communication, Control, and Computing}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","mla":"Redder, Adrian, et al. “Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law.” <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>, 2022.","short":"A. Redder, A. Ramaswamy, H. Karl, in: Proceedings of the 58th Allerton Conference on Communication, Control, and Computing, 2022."},"year":"2022","has_accepted_license":"1"},{"file":[{"file_size":298926,"file_id":"33237","file_name":"ICCART2022.pdf","access_level":"closed","date_updated":"2022-08-31T07:10:13Z","date_created":"2022-08-31T07:10:13Z","creator":"aredder","success":1,"relation":"main_file","content_type":"application/pdf"}],"status":"public","type":"conference","publication":"Proceedings of the 14th International Conference on Agents and Artificial Intelligence","ddc":["006"],"file_date_updated":"2022-08-31T07:10:13Z","language":[{"iso":"eng"}],"project":[{"_id":"16","name":"SFB 901 - C4: SFB 901 - Subproject C4"},{"name":"NICCI-CN: Netzgewahre Regelung & regelungsgewahre Netze","_id":"24"},{"_id":"1","name":"SFB 901: SFB 901"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"}],"_id":"30793","user_id":"477","department":[{"_id":"75"}],"year":"2022","citation":{"ama":"Redder A, Ramaswamy A, Karl H. Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication. In: <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>. SCITEPRESS - Science and Technology Publications; 2022. doi:<a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication,” 2022, doi: <a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Multi-Agent Policy Gradient Algorithms for Cyber-Physical Systems with Lossy Communication.” In <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>. SCITEPRESS - Science and Technology Publications, 2022. <a href=\"https://doi.org/10.5220/0010845400003116\">https://doi.org/10.5220/0010845400003116</a>.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication. <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>. <a href=\"https://doi.org/10.5220/0010845400003116\">https://doi.org/10.5220/0010845400003116</a>","short":"A. Redder, A. Ramaswamy, H. Karl, in: Proceedings of the 14th International Conference on Agents and Artificial Intelligence, SCITEPRESS - Science and Technology Publications, 2022.","bibtex":"@inproceedings{Redder_Ramaswamy_Karl_2022, title={Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication}, DOI={<a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>}, booktitle={Proceedings of the 14th International Conference on Agents and Artificial Intelligence}, publisher={SCITEPRESS - Science and Technology Publications}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","mla":"Redder, Adrian, et al. “Multi-Agent Policy Gradient Algorithms for Cyber-Physical Systems with Lossy Communication.” <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>, SCITEPRESS - Science and Technology Publications, 2022, doi:<a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>."},"publication_status":"published","has_accepted_license":"1","title":"Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication","doi":"10.5220/0010845400003116","date_updated":"2022-11-18T09:32:14Z","publisher":"SCITEPRESS - Science and Technology Publications","author":[{"first_name":"Adrian","full_name":"Redder, Adrian","id":"52265","orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder"},{"first_name":"Arunselvan","full_name":"Ramaswamy, Arunselvan","id":"66937","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111"},{"first_name":"Holger","id":"126","full_name":"Karl, Holger","last_name":"Karl"}],"date_created":"2022-04-06T07:18:36Z"},{"year":"2022","citation":{"chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Distributed Gradient-Based Optimization in the Presence of Dependent  Aperiodic Communication.” <i>ArXiv:2201.11343</i>, 2022.","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Distributed gradient-based optimization in the presence of dependent  aperiodic communication,” <i>arXiv:2201.11343</i>. 2022.","ama":"Redder A, Ramaswamy A, Karl H. Distributed gradient-based optimization in the presence of dependent  aperiodic communication. <i>arXiv:220111343</i>. Published online 2022.","bibtex":"@article{Redder_Ramaswamy_Karl_2022, title={Distributed gradient-based optimization in the presence of dependent  aperiodic communication}, journal={arXiv:2201.11343}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","short":"A. Redder, A. Ramaswamy, H. Karl, ArXiv:2201.11343 (2022).","mla":"Redder, Adrian, et al. “Distributed Gradient-Based Optimization in the Presence of Dependent  Aperiodic Communication.” <i>ArXiv:2201.11343</i>, 2022.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Distributed gradient-based optimization in the presence of dependent  aperiodic communication. In <i>arXiv:2201.11343</i>."},"title":"Distributed gradient-based optimization in the presence of dependent  aperiodic communication","date_updated":"2022-11-18T09:33:01Z","date_created":"2022-04-06T06:53:38Z","author":[{"first_name":"Adrian","orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","full_name":"Redder, Adrian","id":"52265"},{"last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111","id":"66937","full_name":"Ramaswamy, Arunselvan","first_name":"Arunselvan"},{"last_name":"Karl","full_name":"Karl, Holger","id":"126","first_name":"Holger"}],"abstract":[{"text":"Iterative distributed optimization algorithms involve multiple agents that\r\ncommunicate with each other, over time, in order to minimize/maximize a global\r\nobjective. In the presence of unreliable communication networks, the\r\nAge-of-Information (AoI), which measures the freshness of data received, may be\r\nlarge and hence hinder algorithmic convergence. In this paper, we study the\r\nconvergence of general distributed gradient-based optimization algorithms in\r\nthe presence of communication that neither happens periodically nor at\r\nstochastically independent points in time. We show that convergence is\r\nguaranteed provided the random variables associated with the AoI processes are\r\nstochastically dominated by a random variable with finite first moment. This\r\nimproves on previous requirements of boundedness of more than the first moment.\r\nWe then introduce stochastically strongly connected (SSC) networks, a new\r\nstochastic form of strong connectedness for time-varying networks. We show: If\r\nfor any $p \\ge0$ the processes that describe the success of communication\r\nbetween agents in a SSC network are $\\alpha$-mixing with $n^{p-1}\\alpha(n)$\r\nsummable, then the associated AoI processes are stochastically dominated by a\r\nrandom variable with finite $p$-th moment. In combination with our first\r\ncontribution, this implies that distributed stochastic gradient descend\r\nconverges in the presence of AoI, if $\\alpha(n)$ is summable.","lang":"eng"}],"status":"public","publication":"arXiv:2201.11343","type":"preprint","language":[{"iso":"eng"}],"_id":"30790","external_id":{"arxiv":["2201.11343"]},"project":[{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"},{"name":"SFB 901: SFB 901","_id":"1"},{"_id":"4","name":"SFB 901 - C: SFB 901 - Project Area C"}],"department":[{"_id":"75"}],"user_id":"477"},{"type":"preprint","publication":"arXiv:2201.00570","abstract":[{"text":"We present sufficient conditions that ensure convergence of the multi-agent\r\nDeep Deterministic Policy Gradient (DDPG) algorithm. It is an example of one of\r\nthe most popular paradigms of Deep Reinforcement Learning (DeepRL) for tackling\r\ncontinuous action spaces: the actor-critic paradigm. In the setting considered\r\nherein, each agent observes a part of the global state space in order to take\r\nlocal actions, for which it receives local rewards. For every agent, DDPG\r\ntrains a local actor (policy) and a local critic (Q-function). The analysis\r\nshows that multi-agent DDPG using neural networks to approximate the local\r\npolicies and critics converge to limits with the following properties: The\r\ncritic limits minimize the average squared Bellman loss; the actor limits\r\nparameterize a policy that maximizes the local critic's approximation of\r\n$Q_i^*$, where $i$ is the agent index. The averaging is with respect to a\r\nprobability distribution over the global state-action space. It captures the\r\nasymptotics of all local training processes. Finally, we extend the analysis to\r\na fully decentralized setting where agents communicate over a wireless network\r\nprone to delays and losses; a typical scenario in, e.g., robotic applications.","lang":"eng"}],"status":"public","project":[{"_id":"16","name":"SFB 901 - C4: SFB 901 - Subproject C4"},{"_id":"1","name":"SFB 901: SFB 901"},{"_id":"4","name":"SFB 901 - C: SFB 901 - Project Area C"}],"_id":"30791","external_id":{"arxiv":["2201.00570"]},"user_id":"477","department":[{"_id":"75"}],"language":[{"iso":"eng"}],"year":"2022","citation":{"ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms,” <i>arXiv:2201.00570</i>. 2022.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms.” <i>ArXiv:2201.00570</i>, 2022.","ama":"Redder A, Ramaswamy A, Karl H. Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms. <i>arXiv:220100570</i>. Published online 2022.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms. In <i>arXiv:2201.00570</i>.","bibtex":"@article{Redder_Ramaswamy_Karl_2022, title={Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms}, journal={arXiv:2201.00570}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","short":"A. Redder, A. Ramaswamy, H. Karl, ArXiv:2201.00570 (2022).","mla":"Redder, Adrian, et al. “Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms.” <i>ArXiv:2201.00570</i>, 2022."},"date_updated":"2022-11-18T09:33:42Z","date_created":"2022-04-06T06:53:52Z","author":[{"last_name":"Redder","orcid":"https://orcid.org/0000-0001-7391-4688","id":"52265","full_name":"Redder, Adrian","first_name":"Adrian"},{"first_name":"Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111","id":"66937","full_name":"Ramaswamy, Arunselvan"},{"id":"126","full_name":"Karl, Holger","last_name":"Karl","first_name":"Holger"}],"title":"Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms"},{"ddc":["006"],"language":[{"iso":"eng"}],"file":[{"file_size":298395,"access_level":"closed","file_name":"NecSys2022____Practical_Conditions_for_Conv.pdf","file_id":"33236","date_updated":"2022-08-31T07:06:30Z","creator":"aredder","date_created":"2022-08-31T07:06:30Z","success":1,"relation":"main_file","content_type":"application/pdf"}],"publication":"IFAC-PapersOnLine","title":"Practical Network Conditions for the Convergence of Distributed Optimization","publisher":"Elsevier","date_created":"2022-08-16T09:12:55Z","year":"2022","issue":"13","file_date_updated":"2022-08-31T07:06:30Z","_id":"32854","project":[{"_id":"16","name":"SFB 901 - C4: SFB 901 - Subproject C4"},{"name":"SFB 901: SFB 901","_id":"1"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"}],"department":[{"_id":"75"}],"user_id":"477","status":"public","type":"journal_article","conference":{"name":"IFAC Conference on Networked Systems"},"date_updated":"2022-11-18T10:05:14Z","volume":55,"author":[{"id":"52265","full_name":"Redder, Adrian","orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","first_name":"Adrian"},{"first_name":"Arunselvan","id":"66937","full_name":"Ramaswamy, Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111"},{"full_name":"Karl, Holger","id":"126","last_name":"Karl","first_name":"Holger"}],"page":"133–138","intvolume":"        55","citation":{"apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Practical Network Conditions for the Convergence of Distributed Optimization. <i>IFAC-PapersOnLine</i>, <i>55</i>(13), 133–138.","mla":"Redder, Adrian, et al. “Practical Network Conditions for the Convergence of Distributed Optimization.” <i>IFAC-PapersOnLine</i>, vol. 55, no. 13, Elsevier, 2022, pp. 133–138.","bibtex":"@article{Redder_Ramaswamy_Karl_2022, title={Practical Network Conditions for the Convergence of Distributed Optimization}, volume={55}, number={13}, journal={IFAC-PapersOnLine}, publisher={Elsevier}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022}, pages={133–138} }","short":"A. Redder, A. Ramaswamy, H. Karl, IFAC-PapersOnLine 55 (2022) 133–138.","ama":"Redder A, Ramaswamy A, Karl H. Practical Network Conditions for the Convergence of Distributed Optimization. <i>IFAC-PapersOnLine</i>. 2022;55(13):133–138.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Practical Network Conditions for the Convergence of Distributed Optimization.” <i>IFAC-PapersOnLine</i> 55, no. 13 (2022): 133–138.","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Practical Network Conditions for the Convergence of Distributed Optimization,” <i>IFAC-PapersOnLine</i>, vol. 55, no. 13, pp. 133–138, 2022."},"has_accepted_license":"1"},{"citation":{"bibtex":"@article{Ramaswamy_Bhatnagar_2021, title={Analyzing Approximate Value Iteration Algorithms}, journal={Mathematics of Operations Research (to appear)}, author={Ramaswamy, Arunselvan and Bhatnagar, Shalabh}, year={2021} }","short":"A. Ramaswamy, S. Bhatnagar, Mathematics of Operations Research (to Appear) (2021).","mla":"Ramaswamy, Arunselvan, and Shalabh Bhatnagar. “Analyzing Approximate Value Iteration Algorithms.” <i>Mathematics of Operations Research (to Appear)</i>, 2021.","apa":"Ramaswamy, A., &#38; Bhatnagar, S. (2021). Analyzing Approximate Value Iteration Algorithms. <i>Mathematics of Operations Research (to Appear)</i>.","chicago":"Ramaswamy, Arunselvan, and Shalabh Bhatnagar. “Analyzing Approximate Value Iteration Algorithms.” <i>Mathematics of Operations Research (to Appear)</i>, 2021.","ieee":"A. Ramaswamy and S. Bhatnagar, “Analyzing Approximate Value Iteration Algorithms,” <i>Mathematics of Operations Research (to appear)</i>, 2021.","ama":"Ramaswamy A, Bhatnagar S. Analyzing Approximate Value Iteration Algorithms. <i>Mathematics of Operations Research (to appear)</i>. Published online 2021."},"year":"2021","title":"Analyzing Approximate Value Iteration Algorithms","author":[{"orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","id":"66937","full_name":"Ramaswamy, Arunselvan","first_name":"Arunselvan"},{"first_name":"Shalabh","last_name":"Bhatnagar","full_name":"Bhatnagar, Shalabh"}],"date_created":"2021-09-10T09:52:53Z","date_updated":"2022-01-06T06:56:08Z","status":"public","type":"journal_article","publication":"Mathematics of Operations Research (to appear)","language":[{"iso":"eng"}],"user_id":"66937","_id":"24142"},{"language":[{"iso":"eng"}],"user_id":"83504","department":[{"_id":"632"}],"_id":"24143","status":"public","type":"journal_article","publication":"14th ACM Workshop on Artificial Intelligence and Security","title":"Automated Detection of Side Channels in Cryptographic Protocols: DROWN the ROBOTs!","date_created":"2021-09-10T09:56:27Z","author":[{"last_name":"Drees","full_name":"Drees, Jan Peter","first_name":"Jan Peter"},{"first_name":"Pritha","last_name":"Gupta","id":"54803","full_name":"Gupta, Pritha"},{"first_name":"Eyke","id":"48129","full_name":"Hüllermeier, Eyke","last_name":"Hüllermeier"},{"last_name":"Jager","full_name":"Jager, Tibor","first_name":"Tibor"},{"first_name":"Alexander","last_name":"Konze","full_name":"Konze, Alexander"},{"first_name":"Claudia","last_name":"Priesterjahn","full_name":"Priesterjahn, Claudia"},{"id":"66937","full_name":"Ramaswamy, Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111","first_name":"Arunselvan"},{"orcid":"0000-0002-3593-7720","last_name":"Somorovsky","full_name":"Somorovsky, Juraj","id":"83504","first_name":"Juraj"}],"date_updated":"2022-01-06T06:56:08Z","citation":{"apa":"Drees, J. P., Gupta, P., Hüllermeier, E., Jager, T., Konze, A., Priesterjahn, C., Ramaswamy, A., &#38; Somorovsky, J. (2021). Automated Detection of Side Channels in Cryptographic Protocols: DROWN the ROBOTs! <i>14th ACM Workshop on Artificial Intelligence and Security</i>.","mla":"Drees, Jan Peter, et al. “Automated Detection of Side Channels in Cryptographic Protocols: DROWN the ROBOTs!” <i>14th ACM Workshop on Artificial Intelligence and Security</i>, 2021.","bibtex":"@article{Drees_Gupta_Hüllermeier_Jager_Konze_Priesterjahn_Ramaswamy_Somorovsky_2021, title={Automated Detection of Side Channels in Cryptographic Protocols: DROWN the ROBOTs!}, journal={14th ACM Workshop on Artificial Intelligence and Security}, author={Drees, Jan Peter and Gupta, Pritha and Hüllermeier, Eyke and Jager, Tibor and Konze, Alexander and Priesterjahn, Claudia and Ramaswamy, Arunselvan and Somorovsky, Juraj}, year={2021} }","short":"J.P. Drees, P. Gupta, E. Hüllermeier, T. Jager, A. Konze, C. Priesterjahn, A. Ramaswamy, J. Somorovsky, 14th ACM Workshop on Artificial Intelligence and Security (2021).","ama":"Drees JP, Gupta P, Hüllermeier E, et al. Automated Detection of Side Channels in Cryptographic Protocols: DROWN the ROBOTs! <i>14th ACM Workshop on Artificial Intelligence and Security</i>. Published online 2021.","ieee":"J. P. Drees <i>et al.</i>, “Automated Detection of Side Channels in Cryptographic Protocols: DROWN the ROBOTs!,” <i>14th ACM Workshop on Artificial Intelligence and Security</i>, 2021.","chicago":"Drees, Jan Peter, Pritha Gupta, Eyke Hüllermeier, Tibor Jager, Alexander Konze, Claudia Priesterjahn, Arunselvan Ramaswamy, and Juraj Somorovsky. “Automated Detection of Side Channels in Cryptographic Protocols: DROWN the ROBOTs!” <i>14th ACM Workshop on Artificial Intelligence and Security</i>, 2021."},"year":"2021"},{"user_id":"66937","_id":"24148","language":[{"iso":"eng"}],"publication":"IEEE Transactions on Artificial Intelligence (to appear)","type":"journal_article","status":"public","date_created":"2021-09-10T10:03:25Z","author":[{"first_name":"Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","full_name":"Ramaswamy, Arunselvan","id":"66937"},{"id":"48129","full_name":"Hüllermeier, Eyke","last_name":"Hüllermeier","first_name":"Eyke"}],"date_updated":"2022-01-06T06:56:08Z","title":"Deep Q-Learning: Theoretical Insights from an Asymptotic Analysis","citation":{"apa":"Ramaswamy, A., &#38; Hüllermeier, E. (2021). Deep Q-Learning: Theoretical Insights from an Asymptotic Analysis. <i>IEEE Transactions on Artificial Intelligence (to Appear)</i>.","mla":"Ramaswamy, Arunselvan, and Eyke Hüllermeier. “Deep Q-Learning: Theoretical Insights from an Asymptotic Analysis.” <i>IEEE Transactions on Artificial Intelligence (to Appear)</i>, 2021.","short":"A. Ramaswamy, E. Hüllermeier, IEEE Transactions on Artificial Intelligence (to Appear) (2021).","bibtex":"@article{Ramaswamy_Hüllermeier_2021, title={Deep Q-Learning: Theoretical Insights from an Asymptotic Analysis}, journal={IEEE Transactions on Artificial Intelligence (to appear)}, author={Ramaswamy, Arunselvan and Hüllermeier, Eyke}, year={2021} }","ieee":"A. Ramaswamy and E. Hüllermeier, “Deep Q-Learning: Theoretical Insights from an Asymptotic Analysis,” <i>IEEE Transactions on Artificial Intelligence (to appear)</i>, 2021.","chicago":"Ramaswamy, Arunselvan, and Eyke Hüllermeier. “Deep Q-Learning: Theoretical Insights from an Asymptotic Analysis.” <i>IEEE Transactions on Artificial Intelligence (to Appear)</i>, 2021.","ama":"Ramaswamy A, Hüllermeier E. Deep Q-Learning: Theoretical Insights from an Asymptotic Analysis. <i>IEEE Transactions on Artificial Intelligence (to appear)</i>. Published online 2021."},"year":"2021"},{"place":"Montreal, Canada","year":"2021","citation":{"ama":"Afifi H, Ramaswamy A, Karl H. Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks. In: <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>. Montreal, Canada; 2021.","chicago":"Afifi, Haitham, Arunselvan Ramaswamy, and Holger Karl. “Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks.” In <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>. Montreal, Canada, 2021.","ieee":"H. Afifi, A. Ramaswamy, and H. Karl, “Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks,” in <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>, 2021.","mla":"Afifi, Haitham, et al. “Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks.” <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>, 2021.","bibtex":"@inproceedings{Afifi_Ramaswamy_Karl_2021, place={Montreal, Canada}, title={Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks}, booktitle={2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)}, author={Afifi, Haitham and Ramaswamy, Arunselvan and Karl, Holger}, year={2021} }","short":"H. Afifi, A. Ramaswamy, H. Karl, in: 2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium), Montreal, Canada, 2021.","apa":"Afifi, H., Ramaswamy, A., &#38; Karl, H. (2021). Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks. In <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>. Montreal, Canada."},"date_updated":"2022-01-06T06:55:00Z","author":[{"first_name":"Haitham","last_name":"Afifi","full_name":"Afifi, Haitham","id":"65718"},{"first_name":"Arunselvan","full_name":"Ramaswamy, Arunselvan","id":"66937","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy"},{"last_name":"Karl","full_name":"Karl, Holger","id":"126","first_name":"Holger"}],"date_created":"2021-03-12T16:02:04Z","title":"Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks","publication":"2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC'21 - IoTSN Symposium)","type":"conference","abstract":[{"lang":"eng","text":"In this work we use autonomous vehicles to improve the performance of Wireless Sensor Networks (WSNs). In contrast to other autonomous vehicle applications, WSNs have two metrics for performance evaluation. First, quality of information (QoI) which is used to measure the quality of sensed data (e.g., measurement uncertainties or signal strength). Second, quality of service (QoS) which is used to measure the network's performance for data forwarding (e.g., delay and packet losses). As a use case, we consider wireless acoustic sensor networks, where a group of speakers move inside a room and there are autonomous vehicles installed with microphones for streaming the audio data. We formulate the problem as a Markov decision problem (MDP) and solve it using Deep-Q-Networks (DQN). Additionally, we compare the performance of DQN solution to two different real-world implementations: speakers holding/passing microphones and microphones being preinstalled in fixed positions. We show that the performance of autonomous vehicles in terms of QoI and QoS is better than the real-world implementation in some scenarios. Moreover, we study the impact of the vehicles speed on the learning process of the DQN solution and show how low speeds degrade the performance. Finally, we compare the DQN solution to a heuristic one and provide theoretical analysis of the performance with respect to dynamic WSNs."}],"status":"public","_id":"21478","project":[{"_id":"27","name":"Akustische Sensornetzwerke - Teilprojekt \"Verteilte akustische Signalverarbeitung über funkbasierte Sensornetzwerke"}],"user_id":"65718","language":[{"iso":"eng"}]},{"citation":{"short":"H. Afifi, A. Ramaswamy, H. Karl, in: 2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021), 2021.","mla":"Afifi, Haitham, et al. “A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks.” <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>, 2021.","bibtex":"@inproceedings{Afifi_Ramaswamy_Karl_2021, title={A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks}, booktitle={2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)}, author={Afifi, Haitham and Ramaswamy, Arunselvan and Karl, Holger}, year={2021} }","apa":"Afifi, H., Ramaswamy, A., &#38; Karl, H. (2021). A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks. In <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>.","chicago":"Afifi, Haitham, Arunselvan Ramaswamy, and Holger Karl. “A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks.” In <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>, 2021.","ieee":"H. Afifi, A. Ramaswamy, and H. Karl, “A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks,” in <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>, 2021.","ama":"Afifi H, Ramaswamy A, Karl H. A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks. In: <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>. ; 2021."},"year":"2021","author":[{"full_name":"Afifi, Haitham","id":"65718","last_name":"Afifi","first_name":"Haitham"},{"id":"66937","full_name":"Ramaswamy, Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","first_name":"Arunselvan"},{"full_name":"Karl, Holger","id":"126","last_name":"Karl","first_name":"Holger"}],"date_created":"2021-03-12T16:03:53Z","date_updated":"2022-01-06T06:55:00Z","title":"A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks","publication":"2021 IEEE 18th Annual Consumer Communications \\& Networking Conference (CCNC) (CCNC 2021)","type":"conference","status":"public","abstract":[{"text":"Two of the most important metrics when developing Wireless Sensor Networks (WSNs) applications are the Quality of Information (QoI) and Quality of Service (QoS). The former is used to specify the quality of the collected data by the sensors (e.g., measurements error or signal's intensity), while the latter defines the network's performance and availability (e.g., packet losses and latency). In this paper, we consider an example of wireless acoustic sensor networks, where we select a subset of microphones for two different objectives. First, we maximize the recording quality under QoS constraints. Second, we apply a trade-off between QoI and QoS. We formulate the problem as a constrained Markov Decision Problem (MDP) and solve it using reinforcement learning (RL). We compare the RL solution to a baseline model and show that in case of QoS-guarantee objective, the RL solution has an optimality gap up to 1\\%. Meanwhile, the RL solution is better than the baseline with improvements up to 23\\%, when using the trade-off objective.","lang":"eng"}],"user_id":"65718","_id":"21479","project":[{"_id":"27","name":"Akustische Sensornetzwerke - Teilprojekt \"Verteilte akustische Signalverarbeitung über funkbasierte Sensornetzwerke"}],"language":[{"iso":"eng"}],"keyword":["reinforcement learning","wireless sensor networks","resource allocation","acoustic sensor networks"]},{"doi":"10.1109/TAC.2021.3108492","title":"Distributed optimization over time-varying networks with stochastic information delays","date_created":"2021-09-10T09:48:55Z","author":[{"first_name":"Arunselvan","id":"66937","full_name":"Ramaswamy, Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111"},{"first_name":"Adrian","orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","id":"52265","full_name":"Redder, Adrian"},{"full_name":"Quevedo, Daniel E.","last_name":"Quevedo","first_name":"Daniel E."}],"date_updated":"2022-08-16T09:16:43Z","page":"1-1","citation":{"apa":"Ramaswamy, A., Redder, A., &#38; Quevedo, D. E. (2021). Distributed optimization over time-varying networks with stochastic information delays. <i>IEEE Transactions on Automatic Control</i>, 1–1. <a href=\"https://doi.org/10.1109/TAC.2021.3108492\">https://doi.org/10.1109/TAC.2021.3108492</a>","bibtex":"@article{Ramaswamy_Redder_Quevedo_2021, title={Distributed optimization over time-varying networks with stochastic information delays}, DOI={<a href=\"https://doi.org/10.1109/TAC.2021.3108492\">10.1109/TAC.2021.3108492</a>}, journal={IEEE Transactions on Automatic Control}, author={Ramaswamy, Arunselvan and Redder, Adrian and Quevedo, Daniel E.}, year={2021}, pages={1–1} }","short":"A. Ramaswamy, A. Redder, D.E. Quevedo, IEEE Transactions on Automatic Control (2021) 1–1.","mla":"Ramaswamy, Arunselvan, et al. “Distributed Optimization over Time-Varying Networks with Stochastic Information Delays.” <i>IEEE Transactions on Automatic Control</i>, 2021, pp. 1–1, doi:<a href=\"https://doi.org/10.1109/TAC.2021.3108492\">10.1109/TAC.2021.3108492</a>.","ama":"Ramaswamy A, Redder A, Quevedo DE. Distributed optimization over time-varying networks with stochastic information delays. <i>IEEE Transactions on Automatic Control</i>. Published online 2021:1-1. doi:<a href=\"https://doi.org/10.1109/TAC.2021.3108492\">10.1109/TAC.2021.3108492</a>","ieee":"A. Ramaswamy, A. Redder, and D. E. Quevedo, “Distributed optimization over time-varying networks with stochastic information delays,” <i>IEEE Transactions on Automatic Control</i>, pp. 1–1, 2021, doi: <a href=\"https://doi.org/10.1109/TAC.2021.3108492\">10.1109/TAC.2021.3108492</a>.","chicago":"Ramaswamy, Arunselvan, Adrian Redder, and Daniel E. Quevedo. “Distributed Optimization over Time-Varying Networks with Stochastic Information Delays.” <i>IEEE Transactions on Automatic Control</i>, 2021, 1–1. <a href=\"https://doi.org/10.1109/TAC.2021.3108492\">https://doi.org/10.1109/TAC.2021.3108492</a>."},"year":"2021","language":[{"iso":"eng"}],"user_id":"52265","_id":"24140","project":[{"name":"NICCI-CN: Netzgewahre Regelung & regelungsgewahre Netze","_id":"24"}],"status":"public","publication":"IEEE Transactions on Automatic Control","type":"journal_article"},{"citation":{"mla":"Ramaswamy, Arunselvan. “DSPG: Decentralized Simultaneous Perturbations Gradient Descent Scheme.” <i>2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP)</i>, 2020, pp. 54–62.","bibtex":"@inproceedings{Ramaswamy_2020, title={DSPG: Decentralized Simultaneous Perturbations Gradient Descent Scheme}, booktitle={2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP)}, author={Ramaswamy, Arunselvan}, year={2020}, pages={54–62} }","short":"A. Ramaswamy, in: 2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP), 2020, pp. 54–62.","apa":"Ramaswamy, A. (2020). DSPG: Decentralized Simultaneous Perturbations Gradient Descent Scheme. <i>2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP)</i>, 54–62.","ieee":"A. Ramaswamy, “DSPG: Decentralized Simultaneous Perturbations Gradient Descent Scheme,” in <i>2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP)</i>, 2020, pp. 54–62.","chicago":"Ramaswamy, Arunselvan. “DSPG: Decentralized Simultaneous Perturbations Gradient Descent Scheme.” In <i>2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP)</i>, 54–62, 2020.","ama":"Ramaswamy A. DSPG: Decentralized Simultaneous Perturbations Gradient Descent Scheme. In: <i>2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP)</i>. ; 2020:54-62."},"page":"54-62","year":"2020","title":"DSPG: Decentralized Simultaneous Perturbations Gradient Descent Scheme","date_created":"2021-09-10T09:58:09Z","author":[{"first_name":"Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111","id":"66937","full_name":"Ramaswamy, Arunselvan"}],"date_updated":"2022-01-06T06:56:08Z","status":"public","type":"conference","publication":"2020 28th Euromicro International Conference on Parallel, Distributed and Network-Based Processing (PDP)","language":[{"iso":"eng"}],"user_id":"66937","_id":"24145"},{"language":[{"iso":"eng"}],"user_id":"66937","_id":"24146","status":"public","type":"conference","publication":"Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020","title":"Constrained Multi-Agent Optimization with Unbounded Information Delay","author":[{"first_name":"Stefan Helmut","id":"39640","full_name":"Heid, Stefan Helmut","orcid":"0000-0002-9461-7372","last_name":"Heid"},{"full_name":"Ramaswamy, Arunselvan","id":"66937","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","first_name":"Arunselvan"},{"full_name":"Hüllermeier, Eyke","id":"48129","last_name":"Hüllermeier","first_name":"Eyke"}],"date_created":"2021-09-10T09:59:16Z","volume":26,"date_updated":"2022-01-06T06:56:08Z","citation":{"ama":"Heid SH, Ramaswamy A, Hüllermeier E. Constrained Multi-Agent Optimization with Unbounded Information Delay. In: <i>Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020</i>. Vol 26. ; 2020:247.","ieee":"S. H. Heid, A. Ramaswamy, and E. Hüllermeier, “Constrained Multi-Agent Optimization with Unbounded Information Delay,” in <i>Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020</i>, 2020, vol. 26, p. 247.","chicago":"Heid, Stefan Helmut, Arunselvan Ramaswamy, and Eyke Hüllermeier. “Constrained Multi-Agent Optimization with Unbounded Information Delay.” In <i>Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020</i>, 26:247, 2020.","short":"S.H. Heid, A. Ramaswamy, E. Hüllermeier, in: Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020, 2020, p. 247.","bibtex":"@inproceedings{Heid_Ramaswamy_Hüllermeier_2020, title={Constrained Multi-Agent Optimization with Unbounded Information Delay}, volume={26}, booktitle={Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020}, author={Heid, Stefan Helmut and Ramaswamy, Arunselvan and Hüllermeier, Eyke}, year={2020}, pages={247} }","mla":"Heid, Stefan Helmut, et al. “Constrained Multi-Agent Optimization with Unbounded Information Delay.” <i>Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020</i>, vol. 26, 2020, p. 247.","apa":"Heid, S. H., Ramaswamy, A., &#38; Hüllermeier, E. (2020). Constrained Multi-Agent Optimization with Unbounded Information Delay. <i>Proceedings-30. Workshop Computational Intelligence: Berlin, 26.-27. November 2020</i>, <i>26</i>, 247."},"intvolume":"        26","page":"247","year":"2020"},{"_id":"24147","user_id":"66937","language":[{"iso":"eng"}],"type":"journal_article","publication":"IEEE Transactions on Automatic Control","status":"public","date_updated":"2022-01-06T06:56:08Z","publisher":"IEEE","author":[{"full_name":"Ramaswamy, Arunselvan","id":"66937","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","first_name":"Arunselvan"},{"first_name":"Shalabh","full_name":"Bhatnagar, Shalabh","last_name":"Bhatnagar"},{"full_name":"Quevedo, Daniel E","last_name":"Quevedo","first_name":"Daniel E"}],"date_created":"2021-09-10T10:01:18Z","title":"Asynchronous stochastic approximations with asymptotically biased errors and deep multi-agent learning","year":"2020","citation":{"ama":"Ramaswamy A, Bhatnagar S, Quevedo DE. Asynchronous stochastic approximations with asymptotically biased errors and deep multi-agent learning. <i>IEEE Transactions on Automatic Control</i>. Published online 2020.","ieee":"A. Ramaswamy, S. Bhatnagar, and D. E. Quevedo, “Asynchronous stochastic approximations with asymptotically biased errors and deep multi-agent learning,” <i>IEEE Transactions on Automatic Control</i>, 2020.","chicago":"Ramaswamy, Arunselvan, Shalabh Bhatnagar, and Daniel E Quevedo. “Asynchronous Stochastic Approximations with Asymptotically Biased Errors and Deep Multi-Agent Learning.” <i>IEEE Transactions on Automatic Control</i>, 2020.","short":"A. Ramaswamy, S. Bhatnagar, D.E. Quevedo, IEEE Transactions on Automatic Control (2020).","mla":"Ramaswamy, Arunselvan, et al. “Asynchronous Stochastic Approximations with Asymptotically Biased Errors and Deep Multi-Agent Learning.” <i>IEEE Transactions on Automatic Control</i>, IEEE, 2020.","bibtex":"@article{Ramaswamy_Bhatnagar_Quevedo_2020, title={Asynchronous stochastic approximations with asymptotically biased errors and deep multi-agent learning}, journal={IEEE Transactions on Automatic Control}, publisher={IEEE}, author={Ramaswamy, Arunselvan and Bhatnagar, Shalabh and Quevedo, Daniel E}, year={2020} }","apa":"Ramaswamy, A., Bhatnagar, S., &#38; Quevedo, D. E. (2020). Asynchronous stochastic approximations with asymptotically biased errors and deep multi-agent learning. <i>IEEE Transactions on Automatic Control</i>."}},{"language":[{"iso":"eng"}],"user_id":"66937","_id":"24141","status":"public","type":"conference","publication":"Science and Information Conference","title":"Multi-stage reinforcement learning for object detection","author":[{"first_name":"Jonas","full_name":"König, Jonas","last_name":"König"},{"last_name":"Malberg","full_name":"Malberg, Simon","first_name":"Simon"},{"full_name":"Martens, Martin","last_name":"Martens","first_name":"Martin"},{"full_name":"Niehaus, Sebastian","last_name":"Niehaus","first_name":"Sebastian"},{"first_name":"Artus","full_name":"Krohn-Grimberghe, Artus","last_name":"Krohn-Grimberghe"},{"full_name":"Ramaswamy, Arunselvan","id":"66937","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","first_name":"Arunselvan"}],"date_created":"2021-09-10T09:50:53Z","date_updated":"2022-01-06T06:56:08Z","citation":{"ama":"König J, Malberg S, Martens M, Niehaus S, Krohn-Grimberghe A, Ramaswamy A. Multi-stage reinforcement learning for object detection. In: <i>Science and Information Conference</i>. ; 2019:178-191.","chicago":"König, Jonas, Simon Malberg, Martin Martens, Sebastian Niehaus, Artus Krohn-Grimberghe, and Arunselvan Ramaswamy. “Multi-Stage Reinforcement Learning for Object Detection.” In <i>Science and Information Conference</i>, 178–91, 2019.","ieee":"J. König, S. Malberg, M. Martens, S. Niehaus, A. Krohn-Grimberghe, and A. Ramaswamy, “Multi-stage reinforcement learning for object detection,” in <i>Science and Information Conference</i>, 2019, pp. 178–191.","apa":"König, J., Malberg, S., Martens, M., Niehaus, S., Krohn-Grimberghe, A., &#38; Ramaswamy, A. (2019). Multi-stage reinforcement learning for object detection. <i>Science and Information Conference</i>, 178–191.","short":"J. König, S. Malberg, M. Martens, S. Niehaus, A. Krohn-Grimberghe, A. Ramaswamy, in: Science and Information Conference, 2019, pp. 178–191.","mla":"König, Jonas, et al. “Multi-Stage Reinforcement Learning for Object Detection.” <i>Science and Information Conference</i>, 2019, pp. 178–91.","bibtex":"@inproceedings{König_Malberg_Martens_Niehaus_Krohn-Grimberghe_Ramaswamy_2019, title={Multi-stage reinforcement learning for object detection}, booktitle={Science and Information Conference}, author={König, Jonas and Malberg, Simon and Martens, Martin and Niehaus, Sebastian and Krohn-Grimberghe, Artus and Ramaswamy, Arunselvan}, year={2019}, pages={178–191} }"},"page":"178-191","year":"2019"},{"publication":"Automatica","abstract":[{"text":"\r\nIn many cyber–physical systems, we encounter the problem of remote state estimation of geo- graphically distributed and remote physical processes. This paper studies the scheduling of sensor transmissions to estimate the states of multiple remote, dynamic processes. Information from the different sensors has to be transmitted to a central gateway over a wireless network for monitoring purposes, where typically fewer wireless channels are available than there are processes to be monitored. For effective estimation at the gateway, the sensors need to be scheduled appropriately, i.e., at each time instant one needs to decide which sensors have network access and which ones do not. To address this scheduling problem, we formulate an associated Markov decision process (MDP). This MDP is then solved using a Deep Q-Network, a recent deep reinforcement learning algorithm that is at once scalable and model-free. We compare our scheduling algorithm to popular scheduling algorithms such as round-robin and reduced-waiting-time, among others. Our algorithm is shown to significantly outperform these algorithms for many example scenario","lang":"eng"}],"file":[{"creator":"hkarl","date_created":"2020-01-31T15:57:50Z","date_updated":"2020-01-31T15:57:50Z","file_name":"leoram20a.pdf","file_id":"15743","access_level":"closed","file_size":"675382","content_type":"application/pdf","relation":"main_file","success":1}],"ddc":["000"],"language":[{"iso":"eng"}],"quality_controlled":"1","year":"2019","date_created":"2020-01-31T15:55:27Z","title":"Deep reinforcement learning for wireless sensor scheduling in cyber–physical systems","type":"journal_article","status":"public","_id":"15741","project":[{"_id":"24","name":"Netzgewahre Regelung & regelungsgewahre Netze"}],"department":[{"_id":"7"},{"_id":"34"},{"_id":"3"},{"_id":"75"},{"_id":"57"}],"user_id":"126","article_number":"108759","file_date_updated":"2020-01-31T15:57:50Z","publication_identifier":{"issn":["0005-1098"]},"has_accepted_license":"1","publication_status":"published","citation":{"apa":"Leong, A. S., Ramaswamy, A., Quevedo, D. E., Karl, H., &#38; Shi, L. (2019). Deep reinforcement learning for wireless sensor scheduling in cyber–physical systems. <i>Automatica</i>. <a href=\"https://doi.org/10.1016/j.automatica.2019.108759\">https://doi.org/10.1016/j.automatica.2019.108759</a>","short":"A.S. Leong, A. Ramaswamy, D.E. Quevedo, H. Karl, L. Shi, Automatica (2019).","bibtex":"@article{Leong_Ramaswamy_Quevedo_Karl_Shi_2019, title={Deep reinforcement learning for wireless sensor scheduling in cyber–physical systems}, DOI={<a href=\"https://doi.org/10.1016/j.automatica.2019.108759\">10.1016/j.automatica.2019.108759</a>}, number={108759}, journal={Automatica}, author={Leong, Alex S. and Ramaswamy, Arunselvan and Quevedo, Daniel E. and Karl, Holger and Shi, Ling}, year={2019} }","mla":"Leong, Alex S., et al. “Deep Reinforcement Learning for Wireless Sensor Scheduling in Cyber–Physical Systems.” <i>Automatica</i>, 108759, 2019, doi:<a href=\"https://doi.org/10.1016/j.automatica.2019.108759\">10.1016/j.automatica.2019.108759</a>.","ama":"Leong AS, Ramaswamy A, Quevedo DE, Karl H, Shi L. Deep reinforcement learning for wireless sensor scheduling in cyber–physical systems. <i>Automatica</i>. 2019. doi:<a href=\"https://doi.org/10.1016/j.automatica.2019.108759\">10.1016/j.automatica.2019.108759</a>","chicago":"Leong, Alex S., Arunselvan Ramaswamy, Daniel E. Quevedo, Holger Karl, and Ling Shi. “Deep Reinforcement Learning for Wireless Sensor Scheduling in Cyber–Physical Systems.” <i>Automatica</i>, 2019. <a href=\"https://doi.org/10.1016/j.automatica.2019.108759\">https://doi.org/10.1016/j.automatica.2019.108759</a>.","ieee":"A. S. Leong, A. Ramaswamy, D. E. Quevedo, H. Karl, and L. Shi, “Deep reinforcement learning for wireless sensor scheduling in cyber–physical systems,” <i>Automatica</i>, 2019."},"date_updated":"2022-01-06T06:52:32Z","author":[{"last_name":"Leong","full_name":"Leong, Alex S.","first_name":"Alex S."},{"first_name":"Arunselvan","id":"66937","full_name":"Ramaswamy, Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111"},{"last_name":"Quevedo","full_name":"Quevedo, Daniel E.","first_name":"Daniel E."},{"last_name":"Karl","full_name":"Karl, Holger","id":"126","first_name":"Holger"},{"full_name":"Shi, Ling","last_name":"Shi","first_name":"Ling"}],"doi":"10.1016/j.automatica.2019.108759"},{"has_accepted_license":"1","publication_status":"published","citation":{"ieee":"A. Redder, A. Ramaswamy, and D. Quevedo, “Deep reinforcement learning for scheduling in large-scale networked control systems,” in <i>Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems</i>, Chicago, USA, 2019.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Daniel Quevedo. “Deep Reinforcement Learning for Scheduling in Large-Scale Networked Control Systems.” In <i>Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems</i>, 2019.","ama":"Redder A, Ramaswamy A, Quevedo D. Deep reinforcement learning for scheduling in large-scale networked control systems. In: <i>Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems</i>. ; 2019.","apa":"Redder, A., Ramaswamy, A., &#38; Quevedo, D. (2019). Deep reinforcement learning for scheduling in large-scale networked control systems. In <i>Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems</i>. Chicago, USA.","bibtex":"@inproceedings{Redder_Ramaswamy_Quevedo_2019, title={Deep reinforcement learning for scheduling in large-scale networked control systems}, booktitle={Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems}, author={Redder, Adrian and Ramaswamy, Arunselvan and Quevedo, Daniel}, year={2019} }","short":"A. Redder, A. Ramaswamy, D. Quevedo, in: Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems, 2019.","mla":"Redder, Adrian, et al. “Deep Reinforcement Learning for Scheduling in Large-Scale Networked Control Systems.” <i>Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems</i>, 2019."},"oa":"1","date_updated":"2022-01-06T06:51:36Z","author":[{"orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","full_name":"Redder, Adrian","id":"52265","first_name":"Adrian"},{"first_name":"Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","id":"66937","full_name":"Ramaswamy, Arunselvan"},{"full_name":"Quevedo, Daniel","last_name":"Quevedo","first_name":"Daniel"}],"conference":{"name":"8th IFAC Workshop on Distributed Estimation and Control in Networked Systems - NECSYS 2019","start_date":"2019-09-16","end_date":"2019-09-17","location":"Chicago, USA"},"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/1905.05992"}],"type":"conference","status":"public","_id":"13443","project":[{"_id":"52","name":"Computing Resources Provided by the Paderborn Center for Parallel Computing"}],"user_id":"52265","file_date_updated":"2019-09-23T16:21:16Z","year":"2019","date_created":"2019-09-23T16:00:58Z","title":"Deep reinforcement learning for scheduling in large-scale networked control systems","publication":"Proceedings of the 8th IFAC Workshop on Distributed Estimation and Control in Networked Systems","abstract":[{"text":"This work considers the problem of control and resource allocation in networked\r\nsystems. To this end, we present DIRA a Deep reinforcement learning based Iterative Resource\r\nAllocation algorithm, which is scalable and control-aware. Our algorithm is tailored towards\r\nlarge-scale problems where control and scheduling need to act jointly to optimize performance.\r\nDIRA can be used to schedule general time-domain optimization based controllers. In the present\r\nwork, we focus on control designs based on suitably adapted linear quadratic regulators. We\r\napply our algorithm to networked systems with correlated fading communication channels. Our\r\nsimulations show that DIRA scales well to large scheduling problems.","lang":"eng"}],"file":[{"date_updated":"2019-09-23T16:21:16Z","date_created":"2019-09-23T15:48:33Z","creator":"aredder","file_size":371429,"file_id":"13444","file_name":"ifacconf.pdf","access_level":"local","content_type":"application/pdf","relation":"main_file"}],"keyword":["Networked control systems","deep reinforcement learning","large-scale systems","resource scheduling","stochastic control"],"ddc":["620"],"language":[{"iso":"eng"}]},{"status":"public","type":"journal_article","publication":"IEEE Transactions on Automatic Control","language":[{"iso":"eng"}],"_id":"24150","user_id":"66937","department":[{"_id":"355"}],"year":"2018","citation":{"mla":"Ramaswamy, Arunselvan, and Shalabh Bhatnagar. “Stability of Stochastic Approximations with ‘Controlled Markov’ Noise and Temporal Difference Learning.” <i>IEEE Transactions on Automatic Control</i>, vol. 64, no. 6, IEEE, 2018, pp. 2614–20.","short":"A. Ramaswamy, S. Bhatnagar, IEEE Transactions on Automatic Control 64 (2018) 2614–2620.","bibtex":"@article{Ramaswamy_Bhatnagar_2018, title={Stability of stochastic approximations with “controlled markov” noise and temporal difference learning}, volume={64}, number={6}, journal={IEEE Transactions on Automatic Control}, publisher={IEEE}, author={Ramaswamy, Arunselvan and Bhatnagar, Shalabh}, year={2018}, pages={2614–2620} }","apa":"Ramaswamy, A., &#38; Bhatnagar, S. (2018). Stability of stochastic approximations with “controlled markov” noise and temporal difference learning. <i>IEEE Transactions on Automatic Control</i>, <i>64</i>(6), 2614–2620.","chicago":"Ramaswamy, Arunselvan, and Shalabh Bhatnagar. “Stability of Stochastic Approximations with ‘Controlled Markov’ Noise and Temporal Difference Learning.” <i>IEEE Transactions on Automatic Control</i> 64, no. 6 (2018): 2614–20.","ieee":"A. Ramaswamy and S. Bhatnagar, “Stability of stochastic approximations with ‘controlled markov’ noise and temporal difference learning,” <i>IEEE Transactions on Automatic Control</i>, vol. 64, no. 6, pp. 2614–2620, 2018.","ama":"Ramaswamy A, Bhatnagar S. Stability of stochastic approximations with “controlled markov” noise and temporal difference learning. <i>IEEE Transactions on Automatic Control</i>. 2018;64(6):2614-2620."},"intvolume":"        64","page":"2614-2620","issue":"6","title":"Stability of stochastic approximations with “controlled markov” noise and temporal difference learning","publisher":"IEEE","date_updated":"2022-01-06T06:56:08Z","author":[{"first_name":"Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","full_name":"Ramaswamy, Arunselvan","id":"66937"},{"first_name":"Shalabh","last_name":"Bhatnagar","full_name":"Bhatnagar, Shalabh"}],"date_created":"2021-09-10T10:17:54Z","volume":64},{"department":[{"_id":"355"}],"user_id":"66937","_id":"24151","language":[{"iso":"eng"}],"publication":"IEEE Control Systems Letters","type":"journal_article","status":"public","volume":2,"date_created":"2021-09-10T10:19:07Z","author":[{"first_name":"Burak","full_name":"Demirel, Burak","last_name":"Demirel"},{"id":"66937","full_name":"Ramaswamy, Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","first_name":"Arunselvan"},{"first_name":"Daniel E","last_name":"Quevedo","full_name":"Quevedo, Daniel E"},{"first_name":"Holger","last_name":"Karl","full_name":"Karl, Holger"}],"publisher":"IEEE","date_updated":"2022-01-06T06:56:08Z","title":"Deepcas: A deep reinforcement learning algorithm for control-aware scheduling","issue":"4","page":"737-742","intvolume":"         2","citation":{"bibtex":"@article{Demirel_Ramaswamy_Quevedo_Karl_2018, title={Deepcas: A deep reinforcement learning algorithm for control-aware scheduling}, volume={2}, number={4}, journal={IEEE Control Systems Letters}, publisher={IEEE}, author={Demirel, Burak and Ramaswamy, Arunselvan and Quevedo, Daniel E and Karl, Holger}, year={2018}, pages={737–742} }","mla":"Demirel, Burak, et al. “Deepcas: A Deep Reinforcement Learning Algorithm for Control-Aware Scheduling.” <i>IEEE Control Systems Letters</i>, vol. 2, no. 4, IEEE, 2018, pp. 737–42.","short":"B. Demirel, A. Ramaswamy, D.E. Quevedo, H. Karl, IEEE Control Systems Letters 2 (2018) 737–742.","apa":"Demirel, B., Ramaswamy, A., Quevedo, D. E., &#38; Karl, H. (2018). Deepcas: A deep reinforcement learning algorithm for control-aware scheduling. <i>IEEE Control Systems Letters</i>, <i>2</i>(4), 737–742.","ama":"Demirel B, Ramaswamy A, Quevedo DE, Karl H. Deepcas: A deep reinforcement learning algorithm for control-aware scheduling. <i>IEEE Control Systems Letters</i>. 2018;2(4):737-742.","ieee":"B. Demirel, A. Ramaswamy, D. E. Quevedo, and H. Karl, “Deepcas: A deep reinforcement learning algorithm for control-aware scheduling,” <i>IEEE Control Systems Letters</i>, vol. 2, no. 4, pp. 737–742, 2018.","chicago":"Demirel, Burak, Arunselvan Ramaswamy, Daniel E Quevedo, and Holger Karl. “Deepcas: A Deep Reinforcement Learning Algorithm for Control-Aware Scheduling.” <i>IEEE Control Systems Letters</i> 2, no. 4 (2018): 737–42."},"year":"2018"},{"issue":"5","year":"2017","citation":{"apa":"Ramaswamy, A., &#38; Bhatnagar, S. (2017). Analysis of gradient descent methods with nondiminishing bounded errors. <i>IEEE Transactions on Automatic Control</i>, <i>63</i>(5), 1465–1471.","mla":"Ramaswamy, Arunselvan, and Shalabh Bhatnagar. “Analysis of Gradient Descent Methods with Nondiminishing Bounded Errors.” <i>IEEE Transactions on Automatic Control</i>, vol. 63, no. 5, IEEE, 2017, pp. 1465–71.","bibtex":"@article{Ramaswamy_Bhatnagar_2017, title={Analysis of gradient descent methods with nondiminishing bounded errors}, volume={63}, number={5}, journal={IEEE Transactions on Automatic Control}, publisher={IEEE}, author={Ramaswamy, Arunselvan and Bhatnagar, Shalabh}, year={2017}, pages={1465–1471} }","short":"A. Ramaswamy, S. Bhatnagar, IEEE Transactions on Automatic Control 63 (2017) 1465–1471.","chicago":"Ramaswamy, Arunselvan, and Shalabh Bhatnagar. “Analysis of Gradient Descent Methods with Nondiminishing Bounded Errors.” <i>IEEE Transactions on Automatic Control</i> 63, no. 5 (2017): 1465–71.","ieee":"A. Ramaswamy and S. Bhatnagar, “Analysis of gradient descent methods with nondiminishing bounded errors,” <i>IEEE Transactions on Automatic Control</i>, vol. 63, no. 5, pp. 1465–1471, 2017.","ama":"Ramaswamy A, Bhatnagar S. Analysis of gradient descent methods with nondiminishing bounded errors. <i>IEEE Transactions on Automatic Control</i>. 2017;63(5):1465-1471."},"intvolume":"        63","page":"1465-1471","date_updated":"2022-01-06T06:56:08Z","publisher":"IEEE","date_created":"2021-09-10T10:19:40Z","author":[{"last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111","full_name":"Ramaswamy, Arunselvan","id":"66937","first_name":"Arunselvan"},{"full_name":"Bhatnagar, Shalabh","last_name":"Bhatnagar","first_name":"Shalabh"}],"volume":63,"title":"Analysis of gradient descent methods with nondiminishing bounded errors","type":"journal_article","publication":"IEEE Transactions on Automatic Control","status":"public","_id":"24152","user_id":"66937","department":[{"_id":"355"}],"language":[{"iso":"eng"}],"extern":"1"}]