[{"language":[{"iso":"eng"}],"ddc":["004"],"file":[{"creator":"florida","date_created":"2023-07-07T08:24:20Z","date_updated":"2023-07-07T11:16:52Z","access_level":"open_access","file_name":"C4-Chapter-SFB-Buch-Final.pdf","file_id":"45896","file_size":1803186,"content_type":"application/pdf","relation":"main_file"}],"publication":"On-The-Fly Computing -- Individualized IT-services in dynamic markets","title":"On-The-Fly Compute Centers II: Execution of Composed Services in Configurable Compute Centers","date_created":"2023-07-07T08:24:28Z","publisher":"Heinz Nixdorf Institut, Universität Paderborn","year":"2023","file_date_updated":"2023-07-07T11:16:52Z","user_id":"477","series_title":"Verlagsschriftenreihe des Heinz Nixdorf Instituts","department":[{"_id":"7"}],"project":[{"grant_number":"160364472","name":"SFB 901: SFB 901: On-The-Fly Computing - Individualisierte IT-Dienstleistungen in dynamischen Märkten ","_id":"1"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"},{"grant_number":"160364472","_id":"16","name":"SFB 901 - C4: SFB 901 - On-The-Fly Compute Centers II: Ausführung komponierter Dienste in konfigurierbaren Rechenzentren (Subproject C4)"}],"_id":"45895","status":"public","editor":[{"first_name":"Claus-Jochen","full_name":"Haake, Claus-Jochen","last_name":"Haake"},{"first_name":"Friedhelm","last_name":"Meyer auf der Heide","full_name":"Meyer auf der Heide, Friedhelm"},{"last_name":"Platzner","full_name":"Platzner, Marco","first_name":"Marco"},{"first_name":"Henning","last_name":"Wachsmuth","full_name":"Wachsmuth, Henning"},{"first_name":"Heike","last_name":"Wehrheim","full_name":"Wehrheim, Heike"}],"type":"book_chapter","doi":"10.5281/zenodo.8068664","author":[{"id":"126","full_name":"Karl, Holger","last_name":"Karl","first_name":"Holger"},{"first_name":"Marten","id":"88252","full_name":"Maack, Marten","last_name":"Maack"},{"first_name":"Friedhelm","full_name":"Meyer auf der Heide, Friedhelm","id":"15523","last_name":"Meyer auf der Heide"},{"last_name":"Pukrop","id":"44428","full_name":"Pukrop, Simon","first_name":"Simon"},{"orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","id":"52265","full_name":"Redder, Adrian","first_name":"Adrian"}],"volume":412,"date_updated":"2023-07-07T11:16:53Z","oa":"1","citation":{"mla":"Karl, Holger, et al. “On-The-Fly Compute Centers II: Execution of Composed Services in Configurable Compute Centers.” <i>On-The-Fly Computing -- Individualized IT-Services in Dynamic Markets</i>, edited by Claus-Jochen Haake et al., vol. 412, Heinz Nixdorf Institut, Universität Paderborn, 2023, pp. 183–202, doi:<a href=\"https://doi.org/10.5281/zenodo.8068664\">10.5281/zenodo.8068664</a>.","short":"H. Karl, M. Maack, F. Meyer auf der Heide, S. Pukrop, A. Redder, in: C.-J. Haake, F. Meyer auf der Heide, M. Platzner, H. Wachsmuth, H. Wehrheim (Eds.), On-The-Fly Computing -- Individualized IT-Services in Dynamic Markets, Heinz Nixdorf Institut, Universität Paderborn, Paderborn, 2023, pp. 183–202.","bibtex":"@inbook{Karl_Maack_Meyer auf der Heide_Pukrop_Redder_2023, place={Paderborn}, series={Verlagsschriftenreihe des Heinz Nixdorf Instituts}, title={On-The-Fly Compute Centers II: Execution of Composed Services in Configurable Compute Centers}, volume={412}, DOI={<a href=\"https://doi.org/10.5281/zenodo.8068664\">10.5281/zenodo.8068664</a>}, booktitle={On-The-Fly Computing -- Individualized IT-services in dynamic markets}, publisher={Heinz Nixdorf Institut, Universität Paderborn}, author={Karl, Holger and Maack, Marten and Meyer auf der Heide, Friedhelm and Pukrop, Simon and Redder, Adrian}, editor={Haake, Claus-Jochen and Meyer auf der Heide, Friedhelm and Platzner, Marco and Wachsmuth, Henning and Wehrheim, Heike}, year={2023}, pages={183–202}, collection={Verlagsschriftenreihe des Heinz Nixdorf Instituts} }","ama":"Karl H, Maack M, Meyer auf der Heide F, Pukrop S, Redder A. On-The-Fly Compute Centers II: Execution of Composed Services in Configurable Compute Centers. In: Haake C-J, Meyer auf der Heide F, Platzner M, Wachsmuth H, Wehrheim H, eds. <i>On-The-Fly Computing -- Individualized IT-Services in Dynamic Markets</i>. Vol 412. Verlagsschriftenreihe des Heinz Nixdorf Instituts. Heinz Nixdorf Institut, Universität Paderborn; 2023:183-202. doi:<a href=\"https://doi.org/10.5281/zenodo.8068664\">10.5281/zenodo.8068664</a>","apa":"Karl, H., Maack, M., Meyer auf der Heide, F., Pukrop, S., &#38; Redder, A. (2023). On-The-Fly Compute Centers II: Execution of Composed Services in Configurable Compute Centers. In C.-J. Haake, F. Meyer auf der Heide, M. Platzner, H. Wachsmuth, &#38; H. Wehrheim (Eds.), <i>On-The-Fly Computing -- Individualized IT-services in dynamic markets</i> (Vol. 412, pp. 183–202). Heinz Nixdorf Institut, Universität Paderborn. <a href=\"https://doi.org/10.5281/zenodo.8068664\">https://doi.org/10.5281/zenodo.8068664</a>","chicago":"Karl, Holger, Marten Maack, Friedhelm Meyer auf der Heide, Simon Pukrop, and Adrian Redder. “On-The-Fly Compute Centers II: Execution of Composed Services in Configurable Compute Centers.” In <i>On-The-Fly Computing -- Individualized IT-Services in Dynamic Markets</i>, edited by Claus-Jochen Haake, Friedhelm Meyer auf der Heide, Marco Platzner, Henning Wachsmuth, and Heike Wehrheim, 412:183–202. Verlagsschriftenreihe Des Heinz Nixdorf Instituts. Paderborn: Heinz Nixdorf Institut, Universität Paderborn, 2023. <a href=\"https://doi.org/10.5281/zenodo.8068664\">https://doi.org/10.5281/zenodo.8068664</a>.","ieee":"H. Karl, M. Maack, F. Meyer auf der Heide, S. Pukrop, and A. Redder, “On-The-Fly Compute Centers II: Execution of Composed Services in Configurable Compute Centers,” in <i>On-The-Fly Computing -- Individualized IT-services in dynamic markets</i>, vol. 412, C.-J. Haake, F. Meyer auf der Heide, M. Platzner, H. Wachsmuth, and H. Wehrheim, Eds. Paderborn: Heinz Nixdorf Institut, Universität Paderborn, 2023, pp. 183–202."},"page":"183-202","intvolume":"       412","place":"Paderborn","has_accepted_license":"1"},{"date_updated":"2022-03-10T18:28:19Z","oa":"1","author":[{"first_name":"Stefan Balthasar","last_name":"Schneider","orcid":"0000-0001-8210-4011","full_name":"Schneider, Stefan Balthasar","id":"35343"},{"first_name":"Stefan","last_name":"Werner","full_name":"Werner, Stefan"},{"first_name":"Ramin","last_name":"Khalili","full_name":"Khalili, Ramin"},{"last_name":"Hecker","full_name":"Hecker, Artur","first_name":"Artur"},{"last_name":"Karl","full_name":"Karl, Holger","id":"126","first_name":"Holger"}],"conference":{"start_date":"2022-04-25","name":"IEEE/IFIP Network Operations and Management Symposium (NOMS)","location":"Budapest","end_date":"2022-04-29"},"has_accepted_license":"1","citation":{"ieee":"S. B. Schneider, S. Werner, R. Khalili, A. Hecker, and H. Karl, “mobile-env: An Open Platform for Reinforcement Learning in Wireless Mobile Networks,” presented at the IEEE/IFIP Network Operations and Management Symposium (NOMS), Budapest, 2022.","chicago":"Schneider, Stefan Balthasar, Stefan Werner, Ramin Khalili, Artur Hecker, and Holger Karl. “Mobile-Env: An Open Platform for Reinforcement Learning in Wireless Mobile Networks.” In <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>. IEEE, 2022.","ama":"Schneider SB, Werner S, Khalili R, Hecker A, Karl H. mobile-env: An Open Platform for Reinforcement Learning in Wireless Mobile Networks. In: <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>. IEEE; 2022.","mla":"Schneider, Stefan Balthasar, et al. “Mobile-Env: An Open Platform for Reinforcement Learning in Wireless Mobile Networks.” <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>, IEEE, 2022.","short":"S.B. Schneider, S. Werner, R. Khalili, A. Hecker, H. Karl, in: IEEE/IFIP Network Operations and Management Symposium (NOMS), IEEE, 2022.","bibtex":"@inproceedings{Schneider_Werner_Khalili_Hecker_Karl_2022, title={mobile-env: An Open Platform for Reinforcement Learning in Wireless Mobile Networks}, booktitle={IEEE/IFIP Network Operations and Management Symposium (NOMS)}, publisher={IEEE}, author={Schneider, Stefan Balthasar and Werner, Stefan and Khalili, Ramin and Hecker, Artur and Karl, Holger}, year={2022} }","apa":"Schneider, S. B., Werner, S., Khalili, R., Hecker, A., &#38; Karl, H. (2022). mobile-env: An Open Platform for Reinforcement Learning in Wireless Mobile Networks. <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>. IEEE/IFIP Network Operations and Management Symposium (NOMS), Budapest."},"_id":"30236","project":[{"_id":"1","name":"SFB 901: SFB 901"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"},{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"}],"department":[{"_id":"75"}],"user_id":"35343","file_date_updated":"2022-03-10T18:25:41Z","type":"conference","status":"public","publisher":"IEEE","date_created":"2022-03-10T18:28:14Z","title":"mobile-env: An Open Platform for Reinforcement Learning in Wireless Mobile Networks","quality_controlled":"1","year":"2022","keyword":["wireless mobile networks","network management","continuous control","cognitive networks","autonomous coordination","reinforcement learning","gym environment","simulation","open source"],"ddc":["004"],"language":[{"iso":"eng"}],"publication":"IEEE/IFIP Network Operations and Management Symposium (NOMS)","abstract":[{"text":"Recent reinforcement learning approaches for continuous control in wireless mobile networks have shown impressive\r\nresults. But due to the lack of open and compatible simulators, authors typically create their own simulation environments for training and evaluation. This is cumbersome and time-consuming for authors and limits reproducibility and comparability, ultimately impeding progress in the field.\r\n\r\nTo this end, we propose mobile-env, a simple and open platform for training, evaluating, and comparing reinforcement learning and conventional approaches for continuous control in mobile wireless networks. mobile-env is lightweight and implements the common OpenAI Gym interface and additional wrappers, which allows connecting virtually any single-agent or multi-agent reinforcement learning framework to the environment. While mobile-env provides sensible default values and can be used out of the box, it also has many configuration options and is easy to extend. We therefore believe mobile-env to be a valuable platform for driving meaningful progress in autonomous coordination of\r\nwireless mobile networks.","lang":"eng"}],"file":[{"file_size":223412,"access_level":"open_access","file_id":"30237","file_name":"author_version.pdf","date_updated":"2022-03-10T18:25:41Z","date_created":"2022-03-10T18:25:41Z","creator":"stschn","relation":"main_file","content_type":"application/pdf"}]},{"status":"public","abstract":[{"lang":"eng","text":"The decentralized nature of multi-agent systems requires continuous data exchange to achieve global objectives. In such scenarios, Age of Information (AoI) has become an important metric of the freshness of exchanged data due to the error-proneness and delays of communication systems. Communication systems usually possess dependencies: the process describing the success or failure of communication is highly correlated when these attempts are ``close'' in some domain (e.g. in time, frequency, space or code as in wireless communication) and is, in general, non-stationary. To study AoI in such scenarios, we consider an abstract event-based AoI process $\\Delta(n)$, expressing time since the last update: If, at time $n$, a monitoring node receives a status update from a source node (event $A(n-1)$ occurs), then $\\Delta(n)$ is reset to one; otherwise, $\\Delta(n)$ grows linearly in time. This AoI process can thus be viewed as a special random walk with resets. The event process $A(n)$ may be nonstationary and we merely assume that its temporal dependencies decay sufficiently, described by $\\alpha$-mixing. We calculate moment bounds for the resulting AoI process as a function of the mixing rate of $A(n)$. Furthermore, we prove that the AoI process $\\Delta(n)$ is itself $\\alpha$-mixing from which we conclude a strong law of large numbers for $\\Delta(n)$. These results are new, since AoI processes have not been studied so far in this general strongly mixing setting. This opens up future work on renewal processes with non-independent interarrival times."}],"publication":"Proceedings of the 58th Allerton Conference on Communication, Control, and Computing","type":"conference","language":[{"iso":"eng"}],"ddc":["000"],"department":[{"_id":"75"}],"user_id":"477","_id":"32811","project":[{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"},{"_id":"1","name":"SFB 901: SFB 901"},{"_id":"4","name":"SFB 901 - C: SFB 901 - Project Area C"}],"citation":{"mla":"Redder, Adrian, et al. “Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law.” <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>, 2022.","bibtex":"@inproceedings{Redder_Ramaswamy_Karl_2022, title={Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law}, booktitle={Proceedings of the 58th Allerton Conference on Communication, Control, and Computing}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","short":"A. Redder, A. Ramaswamy, H. Karl, in: Proceedings of the 58th Allerton Conference on Communication, Control, and Computing, 2022.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law. <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>. 58th Allerton Conference on Communication, Control, and Computing.","ama":"Redder A, Ramaswamy A, Karl H. Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law. In: <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>. ; 2022.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law.” In <i>Proceedings of the 58th Allerton Conference on Communication, Control, and Computing</i>, 2022.","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law,” presented at the 58th Allerton Conference on Communication, Control, and Computing, 2022."},"year":"2022","has_accepted_license":"1","conference":{"name":"58th Allerton Conference on Communication, Control, and Computing"},"title":"Age of Information Process under Strongly Mixing Communication -- Moment Bound, Mixing Rate and Strong Law","date_created":"2022-08-15T09:59:17Z","author":[{"orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","full_name":"Redder, Adrian","id":"52265","first_name":"Adrian"},{"first_name":"Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","id":"66937","full_name":"Ramaswamy, Arunselvan"},{"first_name":"Holger","id":"126","full_name":"Karl, Holger","last_name":"Karl"}],"date_updated":"2022-11-18T09:31:19Z"},{"file":[{"relation":"main_file","success":1,"content_type":"application/pdf","file_id":"33237","access_level":"closed","file_name":"ICCART2022.pdf","file_size":298926,"date_created":"2022-08-31T07:10:13Z","creator":"aredder","date_updated":"2022-08-31T07:10:13Z"}],"status":"public","type":"conference","publication":"Proceedings of the 14th International Conference on Agents and Artificial Intelligence","ddc":["006"],"language":[{"iso":"eng"}],"file_date_updated":"2022-08-31T07:10:13Z","project":[{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"},{"_id":"24","name":"NICCI-CN: Netzgewahre Regelung & regelungsgewahre Netze"},{"name":"SFB 901: SFB 901","_id":"1"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"}],"_id":"30793","user_id":"477","department":[{"_id":"75"}],"year":"2022","citation":{"ama":"Redder A, Ramaswamy A, Karl H. Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication. In: <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>. SCITEPRESS - Science and Technology Publications; 2022. doi:<a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Multi-Agent Policy Gradient Algorithms for Cyber-Physical Systems with Lossy Communication.” In <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>. SCITEPRESS - Science and Technology Publications, 2022. <a href=\"https://doi.org/10.5220/0010845400003116\">https://doi.org/10.5220/0010845400003116</a>.","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication,” 2022, doi: <a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication. <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>. <a href=\"https://doi.org/10.5220/0010845400003116\">https://doi.org/10.5220/0010845400003116</a>","bibtex":"@inproceedings{Redder_Ramaswamy_Karl_2022, title={Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication}, DOI={<a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>}, booktitle={Proceedings of the 14th International Conference on Agents and Artificial Intelligence}, publisher={SCITEPRESS - Science and Technology Publications}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","short":"A. Redder, A. Ramaswamy, H. Karl, in: Proceedings of the 14th International Conference on Agents and Artificial Intelligence, SCITEPRESS - Science and Technology Publications, 2022.","mla":"Redder, Adrian, et al. “Multi-Agent Policy Gradient Algorithms for Cyber-Physical Systems with Lossy Communication.” <i>Proceedings of the 14th International Conference on Agents and Artificial Intelligence</i>, SCITEPRESS - Science and Technology Publications, 2022, doi:<a href=\"https://doi.org/10.5220/0010845400003116\">10.5220/0010845400003116</a>."},"publication_status":"published","has_accepted_license":"1","title":"Multi-agent Policy Gradient Algorithms for Cyber-physical Systems with Lossy Communication","doi":"10.5220/0010845400003116","date_updated":"2022-11-18T09:32:14Z","publisher":"SCITEPRESS - Science and Technology Publications","date_created":"2022-04-06T07:18:36Z","author":[{"first_name":"Adrian","last_name":"Redder","orcid":"https://orcid.org/0000-0001-7391-4688","id":"52265","full_name":"Redder, Adrian"},{"id":"66937","full_name":"Ramaswamy, Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","first_name":"Arunselvan"},{"full_name":"Karl, Holger","id":"126","last_name":"Karl","first_name":"Holger"}]},{"date_updated":"2022-11-18T09:33:01Z","author":[{"first_name":"Adrian","last_name":"Redder","orcid":"https://orcid.org/0000-0001-7391-4688","full_name":"Redder, Adrian","id":"52265"},{"full_name":"Ramaswamy, Arunselvan","id":"66937","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy","first_name":"Arunselvan"},{"full_name":"Karl, Holger","id":"126","last_name":"Karl","first_name":"Holger"}],"date_created":"2022-04-06T06:53:38Z","title":"Distributed gradient-based optimization in the presence of dependent  aperiodic communication","year":"2022","citation":{"ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Distributed gradient-based optimization in the presence of dependent  aperiodic communication,” <i>arXiv:2201.11343</i>. 2022.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Distributed Gradient-Based Optimization in the Presence of Dependent  Aperiodic Communication.” <i>ArXiv:2201.11343</i>, 2022.","ama":"Redder A, Ramaswamy A, Karl H. Distributed gradient-based optimization in the presence of dependent  aperiodic communication. <i>arXiv:220111343</i>. Published online 2022.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Distributed gradient-based optimization in the presence of dependent  aperiodic communication. In <i>arXiv:2201.11343</i>.","mla":"Redder, Adrian, et al. “Distributed Gradient-Based Optimization in the Presence of Dependent  Aperiodic Communication.” <i>ArXiv:2201.11343</i>, 2022.","bibtex":"@article{Redder_Ramaswamy_Karl_2022, title={Distributed gradient-based optimization in the presence of dependent  aperiodic communication}, journal={arXiv:2201.11343}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","short":"A. Redder, A. Ramaswamy, H. Karl, ArXiv:2201.11343 (2022)."},"project":[{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"},{"_id":"1","name":"SFB 901: SFB 901"},{"_id":"4","name":"SFB 901 - C: SFB 901 - Project Area C"}],"_id":"30790","external_id":{"arxiv":["2201.11343"]},"user_id":"477","department":[{"_id":"75"}],"language":[{"iso":"eng"}],"type":"preprint","publication":"arXiv:2201.11343","abstract":[{"lang":"eng","text":"Iterative distributed optimization algorithms involve multiple agents that\r\ncommunicate with each other, over time, in order to minimize/maximize a global\r\nobjective. In the presence of unreliable communication networks, the\r\nAge-of-Information (AoI), which measures the freshness of data received, may be\r\nlarge and hence hinder algorithmic convergence. In this paper, we study the\r\nconvergence of general distributed gradient-based optimization algorithms in\r\nthe presence of communication that neither happens periodically nor at\r\nstochastically independent points in time. We show that convergence is\r\nguaranteed provided the random variables associated with the AoI processes are\r\nstochastically dominated by a random variable with finite first moment. This\r\nimproves on previous requirements of boundedness of more than the first moment.\r\nWe then introduce stochastically strongly connected (SSC) networks, a new\r\nstochastic form of strong connectedness for time-varying networks. We show: If\r\nfor any $p \\ge0$ the processes that describe the success of communication\r\nbetween agents in a SSC network are $\\alpha$-mixing with $n^{p-1}\\alpha(n)$\r\nsummable, then the associated AoI processes are stochastically dominated by a\r\nrandom variable with finite $p$-th moment. In combination with our first\r\ncontribution, this implies that distributed stochastic gradient descend\r\nconverges in the presence of AoI, if $\\alpha(n)$ is summable."}],"status":"public"},{"date_updated":"2022-11-18T09:33:42Z","author":[{"full_name":"Redder, Adrian","id":"52265","orcid":"https://orcid.org/0000-0001-7391-4688","last_name":"Redder","first_name":"Adrian"},{"id":"66937","full_name":"Ramaswamy, Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111","first_name":"Arunselvan"},{"full_name":"Karl, Holger","id":"126","last_name":"Karl","first_name":"Holger"}],"date_created":"2022-04-06T06:53:52Z","title":"Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms","year":"2022","citation":{"ama":"Redder A, Ramaswamy A, Karl H. Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms. <i>arXiv:220100570</i>. Published online 2022.","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms,” <i>arXiv:2201.00570</i>. 2022.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms.” <i>ArXiv:2201.00570</i>, 2022.","apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms. In <i>arXiv:2201.00570</i>.","bibtex":"@article{Redder_Ramaswamy_Karl_2022, title={Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms}, journal={arXiv:2201.00570}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022} }","short":"A. Redder, A. Ramaswamy, H. Karl, ArXiv:2201.00570 (2022).","mla":"Redder, Adrian, et al. “Asymptotic Convergence of Deep Multi-Agent Actor-Critic Algorithms.” <i>ArXiv:2201.00570</i>, 2022."},"external_id":{"arxiv":["2201.00570"]},"_id":"30791","project":[{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"},{"_id":"1","name":"SFB 901: SFB 901"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"}],"department":[{"_id":"75"}],"user_id":"477","language":[{"iso":"eng"}],"publication":"arXiv:2201.00570","type":"preprint","abstract":[{"text":"We present sufficient conditions that ensure convergence of the multi-agent\r\nDeep Deterministic Policy Gradient (DDPG) algorithm. It is an example of one of\r\nthe most popular paradigms of Deep Reinforcement Learning (DeepRL) for tackling\r\ncontinuous action spaces: the actor-critic paradigm. In the setting considered\r\nherein, each agent observes a part of the global state space in order to take\r\nlocal actions, for which it receives local rewards. For every agent, DDPG\r\ntrains a local actor (policy) and a local critic (Q-function). The analysis\r\nshows that multi-agent DDPG using neural networks to approximate the local\r\npolicies and critics converge to limits with the following properties: The\r\ncritic limits minimize the average squared Bellman loss; the actor limits\r\nparameterize a policy that maximizes the local critic's approximation of\r\n$Q_i^*$, where $i$ is the agent index. The averaging is with respect to a\r\nprobability distribution over the global state-action space. It captures the\r\nasymptotics of all local training processes. Finally, we extend the analysis to\r\na fully decentralized setting where agents communicate over a wireless network\r\nprone to delays and losses; a typical scenario in, e.g., robotic applications.","lang":"eng"}],"status":"public"},{"file_date_updated":"2022-08-31T07:06:30Z","department":[{"_id":"75"}],"user_id":"477","_id":"32854","project":[{"_id":"16","name":"SFB 901 - C4: SFB 901 - Subproject C4"},{"name":"SFB 901: SFB 901","_id":"1"},{"_id":"4","name":"SFB 901 - C: SFB 901 - Project Area C"}],"status":"public","type":"journal_article","conference":{"name":"IFAC Conference on Networked Systems"},"volume":55,"author":[{"first_name":"Adrian","id":"52265","full_name":"Redder, Adrian","last_name":"Redder","orcid":"https://orcid.org/0000-0001-7391-4688"},{"first_name":"Arunselvan","id":"66937","full_name":"Ramaswamy, Arunselvan","last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111"},{"first_name":"Holger","id":"126","full_name":"Karl, Holger","last_name":"Karl"}],"date_updated":"2022-11-18T10:05:14Z","intvolume":"        55","page":"133–138","citation":{"apa":"Redder, A., Ramaswamy, A., &#38; Karl, H. (2022). Practical Network Conditions for the Convergence of Distributed Optimization. <i>IFAC-PapersOnLine</i>, <i>55</i>(13), 133–138.","bibtex":"@article{Redder_Ramaswamy_Karl_2022, title={Practical Network Conditions for the Convergence of Distributed Optimization}, volume={55}, number={13}, journal={IFAC-PapersOnLine}, publisher={Elsevier}, author={Redder, Adrian and Ramaswamy, Arunselvan and Karl, Holger}, year={2022}, pages={133–138} }","mla":"Redder, Adrian, et al. “Practical Network Conditions for the Convergence of Distributed Optimization.” <i>IFAC-PapersOnLine</i>, vol. 55, no. 13, Elsevier, 2022, pp. 133–138.","short":"A. Redder, A. Ramaswamy, H. Karl, IFAC-PapersOnLine 55 (2022) 133–138.","ama":"Redder A, Ramaswamy A, Karl H. Practical Network Conditions for the Convergence of Distributed Optimization. <i>IFAC-PapersOnLine</i>. 2022;55(13):133–138.","ieee":"A. Redder, A. Ramaswamy, and H. Karl, “Practical Network Conditions for the Convergence of Distributed Optimization,” <i>IFAC-PapersOnLine</i>, vol. 55, no. 13, pp. 133–138, 2022.","chicago":"Redder, Adrian, Arunselvan Ramaswamy, and Holger Karl. “Practical Network Conditions for the Convergence of Distributed Optimization.” <i>IFAC-PapersOnLine</i> 55, no. 13 (2022): 133–138."},"has_accepted_license":"1","language":[{"iso":"eng"}],"ddc":["006"],"file":[{"content_type":"application/pdf","success":1,"relation":"main_file","date_updated":"2022-08-31T07:06:30Z","date_created":"2022-08-31T07:06:30Z","creator":"aredder","file_size":298395,"file_name":"NecSys2022____Practical_Conditions_for_Conv.pdf","access_level":"closed","file_id":"33236"}],"publication":"IFAC-PapersOnLine","title":"Practical Network Conditions for the Convergence of Distributed Optimization","date_created":"2022-08-16T09:12:55Z","publisher":"Elsevier","year":"2022","issue":"13"},{"title":"Use What You Know: Network and Service Coordination Beyond Certainty","publisher":"IEEE","date_created":"2022-01-11T08:43:26Z","year":"2022","quality_controlled":"1","keyword":["network management","service management","AI","Monte Carlo Tree Search","model-based","QoS"],"ddc":["004"],"language":[{"iso":"eng"}],"abstract":[{"lang":"eng","text":"Modern services often comprise several components, such as chained virtual network functions, microservices, or\r\nmachine learning functions. Providing such services requires to decide how often to instantiate each component, where to place these instances in the network, how to chain them and route traffic through them. \r\nTo overcome limitations of conventional, hardwired heuristics, deep reinforcement learning (DRL) approaches for self-learning network and service management have emerged recently. These model-free DRL approaches are more flexible but typically learn tabula rasa, i.e., disregard existing understanding of networks, services, and their coordination. \r\n\r\nInstead, we propose FutureCoord, a novel model-based AI approach that leverages existing understanding of networks and services for more efficient and effective coordination without time-intensive training. FutureCoord combines Monte Carlo Tree Search with a stochastic traffic model. This allows FutureCoord to estimate the impact of future incoming traffic and effectively optimize long-term effects, taking fluctuating demand and Quality of Service (QoS) requirements into account. Our extensive evaluation based on real-world network topologies, services, and traffic traces indicates that FutureCoord clearly outperforms state-of-the-art model-free and model-based approaches with up to 51% higher flow success ratios."}],"file":[{"relation":"main_file","content_type":"application/pdf","access_level":"open_access","file_id":"29222","file_name":"author_version.pdf","file_size":528653,"date_created":"2022-01-11T08:39:57Z","creator":"stschn","date_updated":"2022-01-11T08:39:57Z"}],"publication":"IEEE/IFIP Network Operations and Management Symposium (NOMS)","conference":{"start_date":"2022-04-25","name":"IEEE/IFIP Network Operations and Management Symposium (NOMS)","location":"Budapest","end_date":"2022-04-29"},"date_updated":"2022-01-11T08:44:04Z","oa":"1","author":[{"first_name":"Stefan","last_name":"Werner","full_name":"Werner, Stefan"},{"full_name":"Schneider, Stefan Balthasar","id":"35343","orcid":"0000-0001-8210-4011","last_name":"Schneider","first_name":"Stefan Balthasar"},{"first_name":"Holger","id":"126","full_name":"Karl, Holger","last_name":"Karl"}],"citation":{"chicago":"Werner, Stefan, Stefan Balthasar Schneider, and Holger Karl. “Use What You Know: Network and Service Coordination Beyond Certainty.” In <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>. IEEE, 2022.","ieee":"S. Werner, S. B. Schneider, and H. Karl, “Use What You Know: Network and Service Coordination Beyond Certainty,” presented at the IEEE/IFIP Network Operations and Management Symposium (NOMS), Budapest, 2022.","ama":"Werner S, Schneider SB, Karl H. Use What You Know: Network and Service Coordination Beyond Certainty. In: <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>. IEEE; 2022.","mla":"Werner, Stefan, et al. “Use What You Know: Network and Service Coordination Beyond Certainty.” <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>, IEEE, 2022.","bibtex":"@inproceedings{Werner_Schneider_Karl_2022, title={Use What You Know: Network and Service Coordination Beyond Certainty}, booktitle={IEEE/IFIP Network Operations and Management Symposium (NOMS)}, publisher={IEEE}, author={Werner, Stefan and Schneider, Stefan Balthasar and Karl, Holger}, year={2022} }","short":"S. Werner, S.B. Schneider, H. Karl, in: IEEE/IFIP Network Operations and Management Symposium (NOMS), IEEE, 2022.","apa":"Werner, S., Schneider, S. B., &#38; Karl, H. (2022). Use What You Know: Network and Service Coordination Beyond Certainty. <i>IEEE/IFIP Network Operations and Management Symposium (NOMS)</i>. IEEE/IFIP Network Operations and Management Symposium (NOMS), Budapest."},"has_accepted_license":"1","file_date_updated":"2022-01-11T08:39:57Z","_id":"29220","project":[{"_id":"1","name":"SFB 901: SFB 901"},{"name":"SFB 901 - C: SFB 901 - Project Area C","_id":"4"},{"_id":"16","name":"SFB 901 - C4: SFB 901 - Subproject C4"}],"department":[{"_id":"75"}],"user_id":"35343","status":"public","type":"conference"},{"date_updated":"2022-01-06T06:56:58Z","author":[{"last_name":"Afifi","full_name":"Afifi, Haitham","id":"65718","first_name":"Haitham"},{"first_name":"Fabian Jakob","full_name":"Sauer, Fabian Jakob","last_name":"Sauer"},{"first_name":"Holger","last_name":"Karl","full_name":"Karl, Holger","id":"126"}],"date_created":"2021-10-04T10:42:20Z","title":"Reinforcement Learning for Admission Control in Wireless Virtual Network Embedding","has_accepted_license":"1","year":"2021","place":"Hyderabad, India","citation":{"chicago":"Afifi, Haitham, Fabian Jakob Sauer, and Holger Karl. “Reinforcement Learning for Admission Control in Wireless Virtual Network Embedding.” In <i>2021 IEEE International Conference on Advanced Networks and Telecommunications Systems (ANTS) (ANTS’21)</i>. Hyderabad, India, 2021.","ieee":"H. Afifi, F. J. Sauer, and H. Karl, “Reinforcement Learning for Admission Control in Wireless Virtual Network Embedding,” 2021.","ama":"Afifi H, Sauer FJ, Karl H. Reinforcement Learning for Admission Control in Wireless Virtual Network Embedding. In: <i>2021 IEEE International Conference on Advanced Networks and Telecommunications Systems (ANTS) (ANTS’21)</i>. ; 2021.","apa":"Afifi, H., Sauer, F. J., &#38; Karl, H. (2021). Reinforcement Learning for Admission Control in Wireless Virtual Network Embedding. <i>2021 IEEE International Conference on Advanced Networks and Telecommunications Systems (ANTS) (ANTS’21)</i>.","mla":"Afifi, Haitham, et al. “Reinforcement Learning for Admission Control in Wireless Virtual Network Embedding.” <i>2021 IEEE International Conference on Advanced Networks and Telecommunications Systems (ANTS) (ANTS’21)</i>, 2021.","short":"H. Afifi, F.J. Sauer, H. Karl, in: 2021 IEEE International Conference on Advanced Networks and Telecommunications Systems (ANTS) (ANTS’21), Hyderabad, India, 2021.","bibtex":"@inproceedings{Afifi_Sauer_Karl_2021, place={Hyderabad, India}, title={Reinforcement Learning for Admission Control in Wireless Virtual Network Embedding}, booktitle={2021 IEEE International Conference on Advanced Networks and Telecommunications Systems (ANTS) (ANTS’21)}, author={Afifi, Haitham and Sauer, Fabian Jakob and Karl, Holger}, year={2021} }"},"_id":"25278","project":[{"_id":"27","name":"Akustische Sensornetzwerke - Teilprojekt \"Verteilte akustische Signalverarbeitung über funkbasierte Sensornetzwerke"}],"user_id":"65718","keyword":["reinforcement learning","admission control","wireless sensor networks"],"ddc":["000"],"file_date_updated":"2021-10-04T10:43:19Z","language":[{"iso":"eng"}],"publication":"2021 IEEE International Conference on Advanced Networks and Telecommunications Systems (ANTS) (ANTS'21)","type":"conference","abstract":[{"lang":"eng","text":"Using Service Function Chaining (SFC) in wireless networks became popular in many domains like networking and multimedia. It relies on allocating network resources to incoming SFCs requests, via a Virtual Network Embedding (VNE) algorithm, so that it optimizes the performance of the SFC. When the load of incoming requests -- competing for the limited network resources -- increases, it becomes challenging to decide which requests should be admitted and which one should be rejected. In this work, we propose a deep Reinforcement learning (RL) solution that can learn the admission policy for different dependencies, such as the service lifetime and the priority of incoming requests. We compare the deep RL solution to a first-come-first-serve baseline that admits a request whenever there are available resources. We show that deep RL outperforms the baseline and provides higher acceptance rate with low rejections even when there are enough resources."}],"status":"public","file":[{"date_created":"2021-10-04T10:43:19Z","creator":"hafifi","date_updated":"2021-10-04T10:43:19Z","access_level":"closed","file_id":"25279","file_name":"Preprint___Reinforcement_Learning_for_Dynamic_Resource_Allocation_in_Wireless_Networks.pdf","file_size":534737,"content_type":"application/pdf","relation":"main_file","success":1}]},{"status":"public","file":[{"content_type":"application/pdf","success":1,"relation":"main_file","date_updated":"2021-10-04T10:58:07Z","creator":"hafifi","date_created":"2021-10-04T10:58:07Z","file_size":283616,"file_name":"ITG_2021_paper_26 (3).pdf","access_level":"closed","file_id":"25282"}],"abstract":[{"text":"Wireless Acoustic Sensor Networks (WASNs) have a wide range of audio signal processing applications. Due to the spatial diversity of the microphone and their relative position to the acoustic source, not all microphones are equally useful for subsequent audio signal processing tasks, nor do they all have the same wireless data transmission rates. Hence, a central task in WASNs is to balance a microphone’s estimated acoustic utility against its transmission delay, selecting a best-possible subset of microphones to record audio signals.\r\n\r\nIn this work, we use reinforcement learning to decide if a microphone should be used or switched off to maximize the acoustic quality at low transmission delays, while minimizing switching frequency. In experiments with moving sources in a simulated acoustic environment, our method outperforms naive baseline comparisons","lang":"eng"}],"publication":"14. ITG Conference on Speech Communication (ITG 2021)","type":"conference","file_date_updated":"2021-10-04T10:58:07Z","language":[{"iso":"eng"}],"keyword":["microphone utility","microphone selection","wireless acoustic sensor network","network delay","reinforcement learning"],"ddc":["620"],"user_id":"65718","_id":"25281","project":[{"_id":"27","name":"Akustische Sensornetzwerke - Teilprojekt \"Verteilte akustische Signalverarbeitung über funkbasierte Sensornetzwerke"}],"citation":{"apa":"Afifi, H., Guenther, M., Brendel, A., Karl, H., &#38; Kellermann, W. (2021). Reinforcement Learning-based Microphone Selection in Wireless Acoustic Sensor Networks considering Network and Acoustic Utilities. <i>14. ITG Conference on Speech Communication (ITG 2021)</i>.","mla":"Afifi, Haitham, et al. “Reinforcement Learning-Based Microphone Selection in Wireless Acoustic Sensor Networks Considering Network and Acoustic Utilities.” <i>14. ITG Conference on Speech Communication (ITG 2021)</i>, 2021.","bibtex":"@inproceedings{Afifi_Guenther_Brendel_Karl_Kellermann_2021, title={Reinforcement Learning-based Microphone Selection in Wireless Acoustic Sensor Networks considering Network and Acoustic Utilities}, booktitle={14. ITG Conference on Speech Communication (ITG 2021)}, author={Afifi, Haitham and Guenther, Michael and Brendel, Andreas and Karl, Holger and Kellermann, Walter}, year={2021} }","short":"H. Afifi, M. Guenther, A. Brendel, H. Karl, W. Kellermann, in: 14. ITG Conference on Speech Communication (ITG 2021), 2021.","ieee":"H. Afifi, M. Guenther, A. Brendel, H. Karl, and W. Kellermann, “Reinforcement Learning-based Microphone Selection in Wireless Acoustic Sensor Networks considering Network and Acoustic Utilities,” 2021.","chicago":"Afifi, Haitham, Michael Guenther, Andreas Brendel, Holger Karl, and Walter Kellermann. “Reinforcement Learning-Based Microphone Selection in Wireless Acoustic Sensor Networks Considering Network and Acoustic Utilities.” In <i>14. ITG Conference on Speech Communication (ITG 2021)</i>, 2021.","ama":"Afifi H, Guenther M, Brendel A, Karl H, Kellermann W. Reinforcement Learning-based Microphone Selection in Wireless Acoustic Sensor Networks considering Network and Acoustic Utilities. In: <i>14. ITG Conference on Speech Communication (ITG 2021)</i>. ; 2021."},"year":"2021","has_accepted_license":"1","title":"Reinforcement Learning-based Microphone Selection in Wireless Acoustic Sensor Networks considering Network and Acoustic Utilities","date_created":"2021-10-04T10:59:50Z","author":[{"last_name":"Afifi","full_name":"Afifi, Haitham","id":"65718","first_name":"Haitham"},{"first_name":"Michael","last_name":"Guenther","full_name":"Guenther, Michael"},{"first_name":"Andreas","full_name":"Brendel, Andreas","last_name":"Brendel"},{"first_name":"Holger","id":"126","full_name":"Karl, Holger","last_name":"Karl"},{"last_name":"Kellermann","full_name":"Kellermann, Walter","first_name":"Walter"}],"date_updated":"2022-01-06T06:56:59Z"},{"citation":{"ieee":"M. Gunther, H. Afifi, A. Brendel, H. Karl, and W. Kellermann, “Network-Aware Optimal Microphone Channel Selection in Wireless Acoustic Sensor Networks,” 2021, doi: <a href=\"https://doi.org/10.1109/icassp39728.2021.9414528\">10.1109/icassp39728.2021.9414528</a>.","chicago":"Gunther, Michael, Haitham Afifi, Andreas Brendel, Holger Karl, and Walter Kellermann. “Network-Aware Optimal Microphone Channel Selection in Wireless Acoustic Sensor Networks.” In <i>ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)</i>, 2021. <a href=\"https://doi.org/10.1109/icassp39728.2021.9414528\">https://doi.org/10.1109/icassp39728.2021.9414528</a>.","ama":"Gunther M, Afifi H, Brendel A, Karl H, Kellermann W. Network-Aware Optimal Microphone Channel Selection in Wireless Acoustic Sensor Networks. In: <i>ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)</i>. ; 2021. doi:<a href=\"https://doi.org/10.1109/icassp39728.2021.9414528\">10.1109/icassp39728.2021.9414528</a>","apa":"Gunther, M., Afifi, H., Brendel, A., Karl, H., &#38; Kellermann, W. (2021). Network-Aware Optimal Microphone Channel Selection in Wireless Acoustic Sensor Networks. <i>ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)</i>. <a href=\"https://doi.org/10.1109/icassp39728.2021.9414528\">https://doi.org/10.1109/icassp39728.2021.9414528</a>","short":"M. Gunther, H. Afifi, A. Brendel, H. Karl, W. Kellermann, in: ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), 2021.","bibtex":"@inproceedings{Gunther_Afifi_Brendel_Karl_Kellermann_2021, title={Network-Aware Optimal Microphone Channel Selection in Wireless Acoustic Sensor Networks}, DOI={<a href=\"https://doi.org/10.1109/icassp39728.2021.9414528\">10.1109/icassp39728.2021.9414528</a>}, booktitle={ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, author={Gunther, Michael and Afifi, Haitham and Brendel, Andreas and Karl, Holger and Kellermann, Walter}, year={2021} }","mla":"Gunther, Michael, et al. “Network-Aware Optimal Microphone Channel Selection in Wireless Acoustic Sensor Networks.” <i>ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)</i>, 2021, doi:<a href=\"https://doi.org/10.1109/icassp39728.2021.9414528\">10.1109/icassp39728.2021.9414528</a>."},"year":"2021","publication_status":"published","doi":"10.1109/icassp39728.2021.9414528","title":"Network-Aware Optimal Microphone Channel Selection in Wireless Acoustic Sensor Networks","date_created":"2021-10-04T12:28:40Z","author":[{"full_name":"Gunther, Michael","last_name":"Gunther","first_name":"Michael"},{"first_name":"Haitham","last_name":"Afifi","full_name":"Afifi, Haitham","id":"65718"},{"first_name":"Andreas","full_name":"Brendel, Andreas","last_name":"Brendel"},{"first_name":"Holger","id":"126","full_name":"Karl, Holger","last_name":"Karl"},{"first_name":"Walter","full_name":"Kellermann, Walter","last_name":"Kellermann"}],"date_updated":"2022-01-06T06:56:59Z","status":"public","type":"conference","publication":"ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","language":[{"iso":"eng"}],"user_id":"65718","project":[{"name":"Akustische Sensornetzwerke - Teilprojekt \"Verteilte akustische Signalverarbeitung über funkbasierte Sensornetzwerke","_id":"27"}],"_id":"25293"},{"publication_status":"accepted","citation":{"ama":"Hasnain A, Karl H. Learning Flow Scheduling. In: <i>2021 IEEE 18th Annual Consumer Communications &#38; Networking Conference (CCNC)</i>. IEEE Computer Society. doi:<a href=\"https://doi.org/10.1109/CCNC49032.2021.9369514\">https://doi.org/10.1109/CCNC49032.2021.9369514</a>","chicago":"Hasnain, Asif, and Holger Karl. “Learning Flow Scheduling.” In <i>2021 IEEE 18th Annual Consumer Communications &#38; Networking Conference (CCNC)</i>. IEEE Computer Society, n.d. <a href=\"https://doi.org/10.1109/CCNC49032.2021.9369514\">https://doi.org/10.1109/CCNC49032.2021.9369514</a>.","ieee":"A. Hasnain and H. Karl, “Learning Flow Scheduling,” in <i>2021 IEEE 18th Annual Consumer Communications &#38; Networking Conference (CCNC)</i>, Las Vegas, USA.","mla":"Hasnain, Asif, and Holger Karl. “Learning Flow Scheduling.” <i>2021 IEEE 18th Annual Consumer Communications &#38; Networking Conference (CCNC)</i>, IEEE Computer Society, doi:<a href=\"https://doi.org/10.1109/CCNC49032.2021.9369514\">https://doi.org/10.1109/CCNC49032.2021.9369514</a>.","bibtex":"@inproceedings{Hasnain_Karl, title={Learning Flow Scheduling}, DOI={<a href=\"https://doi.org/10.1109/CCNC49032.2021.9369514\">https://doi.org/10.1109/CCNC49032.2021.9369514</a>}, booktitle={2021 IEEE 18th Annual Consumer Communications &#38; Networking Conference (CCNC)}, publisher={IEEE Computer Society}, author={Hasnain, Asif and Karl, Holger} }","short":"A. Hasnain, H. Karl, in: 2021 IEEE 18th Annual Consumer Communications &#38; Networking Conference (CCNC), IEEE Computer Society, n.d.","apa":"Hasnain, A., &#38; Karl, H. (n.d.). Learning Flow Scheduling. In <i>2021 IEEE 18th Annual Consumer Communications &#38; Networking Conference (CCNC)</i>. Las Vegas, USA: IEEE Computer Society. <a href=\"https://doi.org/10.1109/CCNC49032.2021.9369514\">https://doi.org/10.1109/CCNC49032.2021.9369514</a>"},"year":"2021","date_created":"2020-10-19T14:27:17Z","author":[{"first_name":"Asif","id":"63288","full_name":"Hasnain, Asif","last_name":"Hasnain"},{"first_name":"Holger","last_name":"Karl","id":"126","full_name":"Karl, Holger"}],"date_updated":"2022-01-06T06:54:20Z","publisher":"IEEE Computer Society","conference":{"name":"2021 IEEE 18th Annual Consumer Communications & Networking Conference (CCNC)","start_date":"2021-01-09","end_date":"2021-01-12","location":"Las Vegas, USA"},"doi":"https://doi.org/10.1109/CCNC49032.2021.9369514","main_file_link":[{"url":"https://ieeexplore.ieee.org/document/9369514"}],"title":"Learning Flow Scheduling","publication":"2021 IEEE 18th Annual Consumer Communications & Networking Conference (CCNC)","type":"conference","status":"public","abstract":[{"lang":"eng","text":"Datacenter applications have different resource requirements from network and developing flow scheduling heuristics for every workload is practically infeasible. In this paper, we show that deep reinforcement learning (RL) can be used to efficiently learn flow scheduling policies for different workloads without manual feature engineering. Specifically, we present LFS, which learns to optimize a high-level performance objective, e.g., maximize the number of flow admissions while meeting the deadlines. The LFS scheduler is trained through deep RL to learn a scheduling policy on continuous online flow arrivals. The evaluation results show that the trained LFS scheduler admits 1.05x more flows than the greedy flow scheduling heuristics under varying network load."}],"department":[{"_id":"75"}],"user_id":"63288","_id":"20125","project":[{"name":"SFB 901 - Project Area C","_id":"4"},{"_id":"16","name":"SFB 901 - Subproject C4"},{"_id":"1","name":"SFB 901"}],"language":[{"iso":"eng"}],"keyword":["Flow scheduling","Deadlines","Reinforcement learning"],"ddc":["000"]},{"citation":{"ama":"Hasnain A, Karl H. Learning Coflow Admissions. In: <i>IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS)</i>. IEEE Communications Society. doi:<a href=\"https://doi.org/10.1109/INFOCOMWKSHPS51825.2021.9484599\">10.1109/INFOCOMWKSHPS51825.2021.9484599</a>","ieee":"A. Hasnain and H. Karl, “Learning Coflow Admissions,” in <i>IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS)</i>, Vancouver BC Canada.","chicago":"Hasnain, Asif, and Holger Karl. “Learning Coflow Admissions.” In <i>IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS)</i>. IEEE Communications Society, n.d. <a href=\"https://doi.org/10.1109/INFOCOMWKSHPS51825.2021.9484599\">https://doi.org/10.1109/INFOCOMWKSHPS51825.2021.9484599</a>.","apa":"Hasnain, A., &#38; Karl, H. (n.d.). Learning Coflow Admissions. In <i>IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS)</i>. Vancouver BC Canada: IEEE Communications Society. <a href=\"https://doi.org/10.1109/INFOCOMWKSHPS51825.2021.9484599\">https://doi.org/10.1109/INFOCOMWKSHPS51825.2021.9484599</a>","short":"A. Hasnain, H. Karl, in: IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS), IEEE Communications Society, n.d.","mla":"Hasnain, Asif, and Holger Karl. “Learning Coflow Admissions.” <i>IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS)</i>, IEEE Communications Society, doi:<a href=\"https://doi.org/10.1109/INFOCOMWKSHPS51825.2021.9484599\">10.1109/INFOCOMWKSHPS51825.2021.9484599</a>.","bibtex":"@inproceedings{Hasnain_Karl, title={Learning Coflow Admissions}, DOI={<a href=\"https://doi.org/10.1109/INFOCOMWKSHPS51825.2021.9484599\">10.1109/INFOCOMWKSHPS51825.2021.9484599</a>}, booktitle={IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS)}, publisher={IEEE Communications Society}, author={Hasnain, Asif and Karl, Holger} }"},"publication_status":"accepted","related_material":{"link":[{"relation":"confirmation","url":"https://ieeexplore.ieee.org/document/9484599"}]},"main_file_link":[{"url":"https://ieeexplore.ieee.org/document/9484599"}],"doi":"10.1109/INFOCOMWKSHPS51825.2021.9484599","conference":{"location":"Vancouver BC Canada","end_date":"2021-05-13","start_date":"2021-05-10","name":"IEEE INFOCOM 2021 - IEEE Conference on Computer Communications"},"date_updated":"2022-01-06T06:54:42Z","author":[{"id":"63288","full_name":"Hasnain, Asif","last_name":"Hasnain","first_name":"Asif"},{"first_name":"Holger","last_name":"Karl","id":"126","full_name":"Karl, Holger"}],"status":"public","type":"conference","project":[{"_id":"16","name":"SFB 901 - Subproject C4"},{"_id":"4","name":"SFB 901 - Project Area C"},{"_id":"1","name":"SFB 901"}],"_id":"21005","user_id":"63288","department":[{"_id":"75"}],"year":"2021","title":"Learning Coflow Admissions","publisher":"IEEE Communications Society","date_created":"2021-01-16T18:24:19Z","abstract":[{"text":"Data-parallel applications are developed using different data programming models, e.g., MapReduce, partition/aggregate. These models represent diverse resource requirements of application in a datacenter network, which can be represented by the coflow abstraction. The conventional method of creating hand-crafted coflow heuristics for admission or scheduling for different workloads is practically infeasible. In this paper, we propose a deep reinforcement learning (DRL)-based coflow admission scheme -- LCS -- that can learn an admission policy for a higher-level performance objective, i.e., maximize successful coflow admissions, without manual feature engineering.  LCS is trained on a production trace, which has online coflow arrivals. The evaluation results show that LCS is able to learn a reasonable admission policy that admits more coflows than state-of-the-art Varys heuristic while meeting their deadlines.","lang":"eng"}],"publication":"IEEE INFOCOM 2021 - IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS)","ddc":["000"],"keyword":["Coflow scheduling","Reinforcement learning","Deadlines"],"language":[{"iso":"eng"}]},{"language":[{"iso":"eng"}],"_id":"21478","project":[{"name":"Akustische Sensornetzwerke - Teilprojekt \"Verteilte akustische Signalverarbeitung über funkbasierte Sensornetzwerke","_id":"27"}],"user_id":"65718","abstract":[{"lang":"eng","text":"In this work we use autonomous vehicles to improve the performance of Wireless Sensor Networks (WSNs). In contrast to other autonomous vehicle applications, WSNs have two metrics for performance evaluation. First, quality of information (QoI) which is used to measure the quality of sensed data (e.g., measurement uncertainties or signal strength). Second, quality of service (QoS) which is used to measure the network's performance for data forwarding (e.g., delay and packet losses). As a use case, we consider wireless acoustic sensor networks, where a group of speakers move inside a room and there are autonomous vehicles installed with microphones for streaming the audio data. We formulate the problem as a Markov decision problem (MDP) and solve it using Deep-Q-Networks (DQN). Additionally, we compare the performance of DQN solution to two different real-world implementations: speakers holding/passing microphones and microphones being preinstalled in fixed positions. We show that the performance of autonomous vehicles in terms of QoI and QoS is better than the real-world implementation in some scenarios. Moreover, we study the impact of the vehicles speed on the learning process of the DQN solution and show how low speeds degrade the performance. Finally, we compare the DQN solution to a heuristic one and provide theoretical analysis of the performance with respect to dynamic WSNs."}],"status":"public","publication":"2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC'21 - IoTSN Symposium)","type":"conference","title":"Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks","date_updated":"2022-01-06T06:55:00Z","author":[{"id":"65718","full_name":"Afifi, Haitham","last_name":"Afifi","first_name":"Haitham"},{"last_name":"Ramaswamy","orcid":"https://orcid.org/ 0000-0001-7547-8111","full_name":"Ramaswamy, Arunselvan","id":"66937","first_name":"Arunselvan"},{"id":"126","full_name":"Karl, Holger","last_name":"Karl","first_name":"Holger"}],"date_created":"2021-03-12T16:02:04Z","place":"Montreal, Canada","year":"2021","citation":{"ieee":"H. Afifi, A. Ramaswamy, and H. Karl, “Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks,” in <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>, 2021.","chicago":"Afifi, Haitham, Arunselvan Ramaswamy, and Holger Karl. “Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks.” In <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>. Montreal, Canada, 2021.","ama":"Afifi H, Ramaswamy A, Karl H. Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks. In: <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>. Montreal, Canada; 2021.","bibtex":"@inproceedings{Afifi_Ramaswamy_Karl_2021, place={Montreal, Canada}, title={Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks}, booktitle={2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)}, author={Afifi, Haitham and Ramaswamy, Arunselvan and Karl, Holger}, year={2021} }","short":"H. Afifi, A. Ramaswamy, H. Karl, in: 2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium), Montreal, Canada, 2021.","mla":"Afifi, Haitham, et al. “Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks.” <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>, 2021.","apa":"Afifi, H., Ramaswamy, A., &#38; Karl, H. (2021). Reinforcement Learning for Autonomous Vehicle Movements in Wireless Sensor Networks. In <i>2021 IEEE International Conference on Communications (ICC): IoT and Sensor Networks Symposium (IEEE ICC’21 - IoTSN Symposium)</i>. Montreal, Canada."}},{"_id":"21479","project":[{"_id":"27","name":"Akustische Sensornetzwerke - Teilprojekt \"Verteilte akustische Signalverarbeitung über funkbasierte Sensornetzwerke"}],"user_id":"65718","keyword":["reinforcement learning","wireless sensor networks","resource allocation","acoustic sensor networks"],"language":[{"iso":"eng"}],"publication":"2021 IEEE 18th Annual Consumer Communications \\& Networking Conference (CCNC) (CCNC 2021)","type":"conference","abstract":[{"lang":"eng","text":"Two of the most important metrics when developing Wireless Sensor Networks (WSNs) applications are the Quality of Information (QoI) and Quality of Service (QoS). The former is used to specify the quality of the collected data by the sensors (e.g., measurements error or signal's intensity), while the latter defines the network's performance and availability (e.g., packet losses and latency). In this paper, we consider an example of wireless acoustic sensor networks, where we select a subset of microphones for two different objectives. First, we maximize the recording quality under QoS constraints. Second, we apply a trade-off between QoI and QoS. We formulate the problem as a constrained Markov Decision Problem (MDP) and solve it using reinforcement learning (RL). We compare the RL solution to a baseline model and show that in case of QoS-guarantee objective, the RL solution has an optimality gap up to 1\\%. Meanwhile, the RL solution is better than the baseline with improvements up to 23\\%, when using the trade-off objective."}],"status":"public","date_updated":"2022-01-06T06:55:00Z","date_created":"2021-03-12T16:03:53Z","author":[{"first_name":"Haitham","id":"65718","full_name":"Afifi, Haitham","last_name":"Afifi"},{"first_name":"Arunselvan","id":"66937","full_name":"Ramaswamy, Arunselvan","orcid":"https://orcid.org/ 0000-0001-7547-8111","last_name":"Ramaswamy"},{"first_name":"Holger","last_name":"Karl","id":"126","full_name":"Karl, Holger"}],"title":"A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks","year":"2021","citation":{"apa":"Afifi, H., Ramaswamy, A., &#38; Karl, H. (2021). A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks. In <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>.","mla":"Afifi, Haitham, et al. “A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks.” <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>, 2021.","short":"H. Afifi, A. Ramaswamy, H. Karl, in: 2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021), 2021.","bibtex":"@inproceedings{Afifi_Ramaswamy_Karl_2021, title={A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks}, booktitle={2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)}, author={Afifi, Haitham and Ramaswamy, Arunselvan and Karl, Holger}, year={2021} }","chicago":"Afifi, Haitham, Arunselvan Ramaswamy, and Holger Karl. “A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks.” In <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>, 2021.","ieee":"H. Afifi, A. Ramaswamy, and H. Karl, “A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks,” in <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>, 2021.","ama":"Afifi H, Ramaswamy A, Karl H. A Reinforcement Learning QoI/QoS-Aware Approach in Acoustic Sensor Networks. In: <i>2021 IEEE 18th Annual Consumer Communications \\&#38; Networking Conference (CCNC) (CCNC 2021)</i>. ; 2021."}},{"abstract":[{"text":"Services often consist of multiple chained components such as microservices in a service mesh, or machine learning functions in a pipeline. Providing these services requires online coordination including scaling the service, placing instance of all components in the network, scheduling traffic to these instances, and routing traffic through the network. Optimized service coordination is still a hard problem due to many influencing factors such as rapidly arriving user demands and limited node and link capacity. Existing approaches to solve the problem are often built on rigid models and assumptions, tailored to specific scenarios. If the scenario changes and the assumptions no longer hold, they easily break and require manual adjustments by experts. Novel self-learning approaches using deep reinforcement learning (DRL) are promising but still have limitations as they only address simplified versions of the problem and are typically centralized and thus do not scale to practical large-scale networks.\r\n\r\nTo address these issues, we propose a distributed self-learning service coordination approach using DRL. After centralized training, we deploy a distributed DRL agent at each node in the network, making fast coordination decisions locally in parallel with the other nodes. Each agent only observes its direct neighbors and does not need global knowledge. Hence, our approach scales independently from the size of the network. In our extensive evaluation using real-world network topologies and traffic traces, we show that our proposed approach outperforms a state-of-the-art conventional heuristic as well as a centralized DRL approach (60% higher throughput on average) while requiring less time per online decision (1 ms).","lang":"eng"}],"file":[{"date_created":"2021-03-18T17:12:56Z","creator":"stschn","date_updated":"2021-03-18T17:12:56Z","file_id":"21544","access_level":"open_access","file_name":"public_author_version.pdf","file_size":606321,"title":"Distributed Online Service Coordination Using Deep Reinforcement Learning","content_type":"application/pdf","relation":"main_file"}],"publication":"IEEE International Conference on Distributed Computing Systems (ICDCS)","keyword":["network management","service management","coordination","reinforcement learning","distributed"],"ddc":["000"],"language":[{"iso":"eng"}],"year":"2021","title":"Distributed Online Service Coordination Using Deep Reinforcement Learning","publisher":"IEEE","date_created":"2021-03-18T17:15:47Z","status":"public","type":"conference","file_date_updated":"2021-03-18T17:12:56Z","_id":"21543","project":[{"_id":"1","name":"SFB 901"},{"_id":"4","name":"SFB 901 - Project Area C"},{"_id":"16","name":"SFB 901 - Subproject C4"}],"department":[{"_id":"75"}],"user_id":"35343","citation":{"ieee":"S. B. Schneider, H. Qarawlus, and H. Karl, “Distributed Online Service Coordination Using Deep Reinforcement Learning,” in <i>IEEE International Conference on Distributed Computing Systems (ICDCS)</i>, Washington, DC, USA, 2021.","chicago":"Schneider, Stefan Balthasar, Haydar Qarawlus, and Holger Karl. “Distributed Online Service Coordination Using Deep Reinforcement Learning.” In <i>IEEE International Conference on Distributed Computing Systems (ICDCS)</i>. IEEE, 2021.","ama":"Schneider SB, Qarawlus H, Karl H. Distributed Online Service Coordination Using Deep Reinforcement Learning. In: <i>IEEE International Conference on Distributed Computing Systems (ICDCS)</i>. IEEE; 2021.","apa":"Schneider, S. B., Qarawlus, H., &#38; Karl, H. (2021). Distributed Online Service Coordination Using Deep Reinforcement Learning. In <i>IEEE International Conference on Distributed Computing Systems (ICDCS)</i>. Washington, DC, USA: IEEE.","short":"S.B. Schneider, H. Qarawlus, H. Karl, in: IEEE International Conference on Distributed Computing Systems (ICDCS), IEEE, 2021.","mla":"Schneider, Stefan Balthasar, et al. “Distributed Online Service Coordination Using Deep Reinforcement Learning.” <i>IEEE International Conference on Distributed Computing Systems (ICDCS)</i>, IEEE, 2021.","bibtex":"@inproceedings{Schneider_Qarawlus_Karl_2021, title={Distributed Online Service Coordination Using Deep Reinforcement Learning}, booktitle={IEEE International Conference on Distributed Computing Systems (ICDCS)}, publisher={IEEE}, author={Schneider, Stefan Balthasar and Qarawlus, Haydar and Karl, Holger}, year={2021} }"},"has_accepted_license":"1","related_material":{"link":[{"relation":"software","url":"https://github.com/ RealVNF/distributed-drl-coordination"}]},"conference":{"location":"Washington, DC, USA","name":"IEEE International Conference on Distributed Computing Systems (ICDCS)"},"date_updated":"2022-01-06T06:55:04Z","oa":"1","author":[{"first_name":"Stefan Balthasar","full_name":"Schneider, Stefan Balthasar","id":"35343","last_name":"Schneider","orcid":"0000-0001-8210-4011"},{"first_name":"Haydar","last_name":"Qarawlus","full_name":"Qarawlus, Haydar"},{"first_name":"Holger","last_name":"Karl","full_name":"Karl, Holger","id":"126"}]},{"file_date_updated":"2020-12-11T08:37:37Z","_id":"20693","project":[{"_id":"1","name":"SFB 901"},{"_id":"4","name":"SFB 901 - Project Area C"},{"name":"SFB 901 - Subproject C4","_id":"16"}],"department":[{"_id":"75"}],"user_id":"35343","status":"public","type":"conference","conference":{"location":"Bordeaux, France","name":"IFIP/IEEE International Symposium on Integrated Network Management (IM)"},"date_updated":"2022-01-06T06:54:32Z","oa":"1","author":[{"first_name":"Stefan Balthasar","last_name":"Schneider","orcid":"0000-0001-8210-4011","id":"35343","full_name":"Schneider, Stefan Balthasar"},{"full_name":"Jürgens, Mirko","last_name":"Jürgens","first_name":"Mirko"},{"last_name":"Karl","id":"126","full_name":"Karl, Holger","first_name":"Holger"}],"citation":{"bibtex":"@inproceedings{Schneider_Jürgens_Karl_2021, title={Divide and Conquer: Hierarchical Network and Service Coordination}, booktitle={IFIP/IEEE International Symposium on Integrated Network Management (IM)}, publisher={IFIP/IEEE}, author={Schneider, Stefan Balthasar and Jürgens, Mirko and Karl, Holger}, year={2021} }","short":"S.B. Schneider, M. Jürgens, H. Karl, in: IFIP/IEEE International Symposium on Integrated Network Management (IM), IFIP/IEEE, 2021.","mla":"Schneider, Stefan Balthasar, et al. “Divide and Conquer: Hierarchical Network and Service Coordination.” <i>IFIP/IEEE International Symposium on Integrated Network Management (IM)</i>, IFIP/IEEE, 2021.","apa":"Schneider, S. B., Jürgens, M., &#38; Karl, H. (2021). Divide and Conquer: Hierarchical Network and Service Coordination. In <i>IFIP/IEEE International Symposium on Integrated Network Management (IM)</i>. Bordeaux, France: IFIP/IEEE.","ieee":"S. B. Schneider, M. Jürgens, and H. Karl, “Divide and Conquer: Hierarchical Network and Service Coordination,” in <i>IFIP/IEEE International Symposium on Integrated Network Management (IM)</i>, Bordeaux, France, 2021.","chicago":"Schneider, Stefan Balthasar, Mirko Jürgens, and Holger Karl. “Divide and Conquer: Hierarchical Network and Service Coordination.” In <i>IFIP/IEEE International Symposium on Integrated Network Management (IM)</i>. IFIP/IEEE, 2021.","ama":"Schneider SB, Jürgens M, Karl H. Divide and Conquer: Hierarchical Network and Service Coordination. In: <i>IFIP/IEEE International Symposium on Integrated Network Management (IM)</i>. IFIP/IEEE; 2021."},"has_accepted_license":"1","keyword":["network management","service management","coordination","hierarchical","scalability","nfv"],"ddc":["006"],"language":[{"iso":"eng"}],"abstract":[{"text":"In practical, large-scale networks, services are requested\r\nby users across the globe, e.g., for video streaming.\r\nServices consist of multiple interconnected components such as\r\nmicroservices in a service mesh. Coordinating these services\r\nrequires scaling them according to continuously changing user\r\ndemand, deploying instances at the edge close to their users,\r\nand routing traffic efficiently between users and connected instances.\r\nNetwork and service coordination is commonly addressed\r\nthrough centralized approaches, where a single coordinator\r\nknows everything and coordinates the entire network globally.\r\nWhile such centralized approaches can reach global optima, they\r\ndo not scale to large, realistic networks. In contrast, distributed\r\napproaches scale well, but sacrifice solution quality due to their\r\nlimited scope of knowledge and coordination decisions.\r\n\r\nTo this end, we propose a hierarchical coordination approach\r\nthat combines the good solution quality of centralized approaches\r\nwith the scalability of distributed approaches. In doing so, we divide\r\nthe network into multiple hierarchical domains and optimize\r\ncoordination in a top-down manner. We compare our hierarchical\r\nwith a centralized approach in an extensive evaluation on a real-world\r\nnetwork topology. Our results indicate that hierarchical\r\ncoordination can find close-to-optimal solutions in a fraction of\r\nthe runtime of centralized approaches.","lang":"eng"}],"file":[{"relation":"main_file","content_type":"application/pdf","access_level":"open_access","file_id":"20694","file_name":"preprint_with_header.pdf","file_size":7979772,"title":"Divide and Conquer: Hierarchical Network and Service Coordination","date_created":"2020-12-11T08:37:37Z","creator":"stschn","date_updated":"2020-12-11T08:37:37Z"}],"publication":"IFIP/IEEE International Symposium on Integrated Network Management (IM)","title":"Divide and Conquer: Hierarchical Network and Service Coordination","publisher":"IFIP/IEEE","date_created":"2020-12-11T08:39:47Z","year":"2021","quality_controlled":"1"},{"doi":"10.1109/TNSM.2021.3076503","title":"Self-Learning Multi-Objective Service Coordination Using Deep Reinforcement Learning","date_created":"2021-04-27T08:04:16Z","author":[{"last_name":"Schneider","orcid":"0000-0001-8210-4011","full_name":"Schneider, Stefan Balthasar","id":"35343","first_name":"Stefan Balthasar"},{"full_name":"Khalili, Ramin","last_name":"Khalili","first_name":"Ramin"},{"first_name":"Adnan","last_name":"Manzoor","full_name":"Manzoor, Adnan"},{"full_name":"Qarawlus, Haydar","last_name":"Qarawlus","first_name":"Haydar"},{"full_name":"Schellenberg, Rafael","last_name":"Schellenberg","first_name":"Rafael"},{"id":"126","full_name":"Karl, Holger","last_name":"Karl","first_name":"Holger"},{"last_name":"Hecker","full_name":"Hecker, Artur","first_name":"Artur"}],"publisher":"IEEE","date_updated":"2022-01-06T06:55:15Z","oa":"1","citation":{"ama":"Schneider SB, Khalili R, Manzoor A, et al. Self-Learning Multi-Objective Service Coordination Using Deep Reinforcement Learning. <i>Transactions on Network and Service Management</i>. 2021. doi:<a href=\"https://doi.org/10.1109/TNSM.2021.3076503\">10.1109/TNSM.2021.3076503</a>","chicago":"Schneider, Stefan Balthasar, Ramin Khalili, Adnan Manzoor, Haydar Qarawlus, Rafael Schellenberg, Holger Karl, and Artur Hecker. “Self-Learning Multi-Objective Service Coordination Using Deep Reinforcement Learning.” <i>Transactions on Network and Service Management</i>, 2021. <a href=\"https://doi.org/10.1109/TNSM.2021.3076503\">https://doi.org/10.1109/TNSM.2021.3076503</a>.","ieee":"S. B. Schneider <i>et al.</i>, “Self-Learning Multi-Objective Service Coordination Using Deep Reinforcement Learning,” <i>Transactions on Network and Service Management</i>, 2021.","apa":"Schneider, S. B., Khalili, R., Manzoor, A., Qarawlus, H., Schellenberg, R., Karl, H., &#38; Hecker, A. (2021). Self-Learning Multi-Objective Service Coordination Using Deep Reinforcement Learning. <i>Transactions on Network and Service Management</i>. <a href=\"https://doi.org/10.1109/TNSM.2021.3076503\">https://doi.org/10.1109/TNSM.2021.3076503</a>","mla":"Schneider, Stefan Balthasar, et al. “Self-Learning Multi-Objective Service Coordination Using Deep Reinforcement Learning.” <i>Transactions on Network and Service Management</i>, IEEE, 2021, doi:<a href=\"https://doi.org/10.1109/TNSM.2021.3076503\">10.1109/TNSM.2021.3076503</a>.","short":"S.B. Schneider, R. Khalili, A. Manzoor, H. Qarawlus, R. Schellenberg, H. Karl, A. Hecker, Transactions on Network and Service Management (2021).","bibtex":"@article{Schneider_Khalili_Manzoor_Qarawlus_Schellenberg_Karl_Hecker_2021, title={Self-Learning Multi-Objective Service Coordination Using Deep Reinforcement Learning}, DOI={<a href=\"https://doi.org/10.1109/TNSM.2021.3076503\">10.1109/TNSM.2021.3076503</a>}, journal={Transactions on Network and Service Management}, publisher={IEEE}, author={Schneider, Stefan Balthasar and Khalili, Ramin and Manzoor, Adnan and Qarawlus, Haydar and Schellenberg, Rafael and Karl, Holger and Hecker, Artur}, year={2021} }"},"year":"2021","has_accepted_license":"1","language":[{"iso":"eng"}],"file_date_updated":"2021-04-27T08:01:26Z","article_type":"original","ddc":["000"],"keyword":["network management","service management","coordination","reinforcement learning","self-learning","self-adaptation","multi-objective"],"user_id":"35343","department":[{"_id":"75"}],"project":[{"name":"SFB 901","_id":"1"},{"name":"SFB 901 - Project Area C","_id":"4"},{"name":"SFB 901 - Subproject C4","_id":"16"}],"_id":"21808","file":[{"date_created":"2021-04-27T08:01:26Z","creator":"stschn","date_updated":"2021-04-27T08:01:26Z","access_level":"open_access","file_name":"ris-accepted-version.pdf","file_id":"21809","description":"Author version of the accepted paper","file_size":4172270,"content_type":"application/pdf","relation":"main_file"}],"status":"public","abstract":[{"text":"Modern services consist of interconnected components,e.g., microservices in a service mesh or machine learning functions in a pipeline. These services can scale and run across multiple network nodes on demand. To process incoming traffic, service components have to be instantiated and traffic assigned to these instances, taking capacities, changing demands, and Quality of Service (QoS) requirements into account. This challenge is usually solved with custom approaches designed by experts. While this typically works well for the considered scenario, the models often rely on unrealistic assumptions or on knowledge that is not available in practice (e.g., a priori knowledge).\r\n\r\nWe propose DeepCoord, a novel deep reinforcement learning approach that learns how to best coordinate services and is geared towards realistic assumptions. It interacts with the network and relies on available, possibly delayed monitoring information. Rather than defining a complex model or an algorithm on how to achieve an objective, our model-free approach adapts to various objectives and traffic patterns. An agent is trained offline without expert knowledge and then applied online with minimal overhead. Compared to a state-of-the-art heuristic, DeepCoord significantly improves flow throughput (up to 76%) and overall network utility (more than 2x) on realworld network topologies and traffic traces. It also supports optimizing multiple, possibly competing objectives, learns to respect QoS requirements, generalizes to scenarios with unseen, stochastic traffic, and scales to large real-world networks. For reproducibility and reuse, our code is publicly available.","lang":"eng"}],"type":"journal_article","publication":"Transactions on Network and Service Management"},{"citation":{"apa":"Schneider, S. B., Karl, H., Khalili, R., &#38; Hecker, A. (2021). <i>DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning</i>.","mla":"Schneider, Stefan Balthasar, et al. <i>DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning</i>. 2021.","short":"S.B. Schneider, H. Karl, R. Khalili, A. Hecker, DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning, 2021.","bibtex":"@book{Schneider_Karl_Khalili_Hecker_2021, title={DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning}, author={Schneider, Stefan Balthasar and Karl, Holger and Khalili, Ramin and Hecker, Artur}, year={2021} }","ama":"Schneider SB, Karl H, Khalili R, Hecker A. <i>DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning</i>.; 2021.","chicago":"Schneider, Stefan Balthasar, Holger Karl, Ramin Khalili, and Artur Hecker. <i>DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning</i>, 2021.","ieee":"S. B. Schneider, H. Karl, R. Khalili, and A. Hecker, <i>DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning</i>. 2021."},"year":"2021","has_accepted_license":"1","title":"DeepCoMP: Coordinated Multipoint Using Multi-Agent Deep Reinforcement Learning","date_created":"2022-10-20T16:44:19Z","author":[{"orcid":"0000-0001-8210-4011","last_name":"Schneider","id":"35343","full_name":"Schneider, Stefan Balthasar","first_name":"Stefan Balthasar"},{"last_name":"Karl","id":"126","full_name":"Karl, Holger","first_name":"Holger"},{"full_name":"Khalili, Ramin","last_name":"Khalili","first_name":"Ramin"},{"last_name":"Hecker","full_name":"Hecker, Artur","first_name":"Artur"}],"date_updated":"2022-11-18T09:59:27Z","oa":"1","file":[{"relation":"main_file","content_type":"application/pdf","access_level":"open_access","file_name":"preprint.pdf","file_id":"33855","file_size":2521656,"date_created":"2022-10-20T16:41:10Z","creator":"stschn","date_updated":"2022-10-20T16:41:10Z"}],"status":"public","abstract":[{"text":"Macrodiversity is a key technique to increase the capacity of mobile networks. It can be realized using coordinated multipoint (CoMP), simultaneously connecting users to multiple overlapping cells. Selecting which users to serve by how many and which cells is NP-hard but needs to happen continuously in real time as users move and channel state changes. Existing approaches often require strict assumptions about or perfect knowledge of the underlying radio system, its resource allocation scheme, or user movements, none of which is readily available in practice.\r\n\r\nInstead, we propose three novel self-learning and self-adapting approaches using model-free deep reinforcement learning (DRL): DeepCoMP, DD-CoMP, and D3-CoMP. DeepCoMP leverages central observations and control of all users to select cells almost optimally. DD-CoMP and D3-CoMP use multi-agent DRL, which allows distributed, robust, and highly scalable coordination. All three approaches learn from experience and self-adapt to varying scenarios, reaching 2x higher Quality of Experience than other approaches. They have very few built-in assumptions and do not need prior system knowledge, making them more robust to change and better applicable in practice than existing approaches.","lang":"eng"}],"type":"working_paper","language":[{"iso":"eng"}],"file_date_updated":"2022-10-20T16:41:10Z","ddc":["004"],"keyword":["mobility management","coordinated multipoint","CoMP","cell selection","resource management","reinforcement learning","multi agent","MARL","self-learning","self-adaptation","QoE"],"user_id":"477","department":[{"_id":"75"}],"project":[{"_id":"4","name":"SFB 901 - C: SFB 901 - Project Area C"},{"name":"SFB 901 - C4: SFB 901 - Subproject C4","_id":"16"},{"_id":"1","name":"SFB 901: SFB 901"}],"_id":"33854"},{"author":[{"first_name":"Stefan Balthasar","orcid":"0000-0001-8210-4011","last_name":"Schneider","id":"35343","full_name":"Schneider, Stefan Balthasar"},{"first_name":"Lars Dietrich","last_name":"Klenner","full_name":"Klenner, Lars Dietrich"},{"first_name":"Holger","last_name":"Karl","full_name":"Karl, Holger","id":"126"}],"date_updated":"2022-01-06T06:54:08Z","oa":"1","has_accepted_license":"1","citation":{"apa":"Schneider, S. B., Klenner, L. D., &#38; Karl, H. (2020). Every Node for Itself: Fully Distributed Service Coordination. In <i>IEEE International Conference on Network and Service Management (CNSM)</i>. IEEE.","mla":"Schneider, Stefan Balthasar, et al. “Every Node for Itself: Fully Distributed Service Coordination.” <i>IEEE International Conference on Network and Service Management (CNSM)</i>, IEEE, 2020.","short":"S.B. Schneider, L.D. Klenner, H. Karl, in: IEEE International Conference on Network and Service Management (CNSM), IEEE, 2020.","bibtex":"@inproceedings{Schneider_Klenner_Karl_2020, title={Every Node for Itself: Fully Distributed Service Coordination}, booktitle={IEEE International Conference on Network and Service Management (CNSM)}, publisher={IEEE}, author={Schneider, Stefan Balthasar and Klenner, Lars Dietrich and Karl, Holger}, year={2020} }","ieee":"S. B. Schneider, L. D. Klenner, and H. Karl, “Every Node for Itself: Fully Distributed Service Coordination,” in <i>IEEE International Conference on Network and Service Management (CNSM)</i>, 2020.","chicago":"Schneider, Stefan Balthasar, Lars Dietrich Klenner, and Holger Karl. “Every Node for Itself: Fully Distributed Service Coordination.” In <i>IEEE International Conference on Network and Service Management (CNSM)</i>. IEEE, 2020.","ama":"Schneider SB, Klenner LD, Karl H. Every Node for Itself: Fully Distributed Service Coordination. In: <i>IEEE International Conference on Network and Service Management (CNSM)</i>. IEEE; 2020."},"user_id":"35343","department":[{"_id":"75"}],"project":[{"_id":"1","name":"SFB 901"},{"_id":"4","name":"SFB 901 - Project Area C"},{"name":"SFB 901 - Subproject C4","_id":"16"}],"_id":"19607","file_date_updated":"2020-09-22T06:36:25Z","type":"conference","status":"public","date_created":"2020-09-22T06:23:40Z","publisher":"IEEE","title":"Every Node for Itself: Fully Distributed Service Coordination","year":"2020","language":[{"iso":"eng"}],"ddc":["006"],"keyword":["distributed management","service coordination","network coordination","nfv","softwarization","orchestration"],"publication":"IEEE International Conference on Network and Service Management (CNSM)","file":[{"date_updated":"2020-09-22T06:36:25Z","creator":"stschn","date_created":"2020-09-22T06:25:57Z","file_size":500948,"access_level":"open_access","file_name":"ris_with_copyright.pdf","file_id":"19608","content_type":"application/pdf","relation":"main_file"}],"abstract":[{"lang":"eng","text":"Modern services consist of modular, interconnected\r\ncomponents, e.g., microservices forming a service mesh. To\r\ndynamically adjust to ever-changing service demands, service\r\ncomponents have to be instantiated on nodes across the network.\r\nIncoming flows requesting a service then need to be routed\r\nthrough the deployed instances while considering node and link\r\ncapacities. Ultimately, the goal is to maximize the successfully\r\nserved flows and Quality of Service (QoS) through online service\r\ncoordination. Current approaches for service coordination are\r\nusually centralized, assuming up-to-date global knowledge and\r\nmaking global decisions for all nodes in the network. Such global\r\nknowledge and centralized decisions are not realistic in practical\r\nlarge-scale networks.\r\n\r\nTo solve this problem, we propose two algorithms for fully\r\ndistributed service coordination. The proposed algorithms can be\r\nexecuted individually at each node in parallel and require only\r\nvery limited global knowledge. We compare and evaluate both\r\nalgorithms with a state-of-the-art centralized approach in extensive\r\nsimulations on a large-scale, real-world network topology.\r\nOur results indicate that the two algorithms can compete with\r\ncentralized approaches in terms of solution quality but require\r\nless global knowledge and are magnitudes faster (more than\r\n100x)."}]}]