[{"abstract":[{"text":"Artificial benchmark functions are commonly used in optimization research because of their ability to rapidly evaluate potential solutions, making them a preferred substitute for real-world problems. However, these benchmark functions have faced criticism for their limited resemblance to real-world problems. In response, recent research has focused on automatically generating new benchmark functions for areas where established test suites are inadequate. These approaches have limitations, such as the difficulty of generating new benchmark functions that exhibit exploratory landscape analysis (ELA) features beyond those of existing benchmarks.The objective of this work is to develop a method for generating benchmark functions for single-objective continuous optimization with user-specified structural properties. Specifically, we aim to demonstrate a proof of concept for a method that uses an ELA feature vector to specify these properties in advance. To achieve this, we begin by generating a random sample of decision space variables and objective values. We then adjust the objective values using CMA-ES until the corresponding features of our new problem match the predefined ELA features within a specified threshold. By iteratively transforming the landscape in this way, we ensure that the resulting function exhibits the desired properties. To create the final function, we use the resulting point cloud as training data for a simple neural network that produces a function exhibiting the target ELA features. We demonstrate the effectiveness of this approach by replicating the existing functions of the well-known BBOB suite and creating new functions with ELA feature values that are not present in BBOB.","lang":"eng"}],"status":"public","type":"conference","publication":"Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms","keyword":["Benchmarking","Instance Generator","Black-Box Continuous Optimization","Exploratory Landscape Analysis","Neural Networks"],"language":[{"iso":"eng"}],"_id":"47522","series_title":"FOGA ’23","user_id":"15504","department":[{"_id":"34"},{"_id":"819"}],"place":"New York, NY, USA","year":"2023","citation":{"mla":"Prager, Raphael Patrick, et al. “Neural Networks as Black-Box Benchmark Functions Optimized for Exploratory Landscape Features.” <i>Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, Association for Computing Machinery, 2023, pp. 129–139, doi:<a href=\"https://doi.org/10.1145/3594805.3607136\">10.1145/3594805.3607136</a>.","short":"R.P. Prager, K. Dietrich, L. Schneider, L. Schäpermeier, B. Bischl, P. Kerschke, H. Trautmann, O. Mersmann, in: Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms, Association for Computing Machinery, New York, NY, USA, 2023, pp. 129–139.","bibtex":"@inproceedings{Prager_Dietrich_Schneider_Schäpermeier_Bischl_Kerschke_Trautmann_Mersmann_2023, place={New York, NY, USA}, series={FOGA ’23}, title={Neural Networks as Black-Box Benchmark Functions Optimized for Exploratory Landscape Features}, DOI={<a href=\"https://doi.org/10.1145/3594805.3607136\">10.1145/3594805.3607136</a>}, booktitle={Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms}, publisher={Association for Computing Machinery}, author={Prager, Raphael Patrick and Dietrich, Konstantin and Schneider, Lennart and Schäpermeier, Lennart and Bischl, Bernd and Kerschke, Pascal and Trautmann, Heike and Mersmann, Olaf}, year={2023}, pages={129–139}, collection={FOGA ’23} }","apa":"Prager, R. P., Dietrich, K., Schneider, L., Schäpermeier, L., Bischl, B., Kerschke, P., Trautmann, H., &#38; Mersmann, O. (2023). Neural Networks as Black-Box Benchmark Functions Optimized for Exploratory Landscape Features. <i>Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, 129–139. <a href=\"https://doi.org/10.1145/3594805.3607136\">https://doi.org/10.1145/3594805.3607136</a>","ama":"Prager RP, Dietrich K, Schneider L, et al. Neural Networks as Black-Box Benchmark Functions Optimized for Exploratory Landscape Features. In: <i>Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>. FOGA ’23. Association for Computing Machinery; 2023:129–139. doi:<a href=\"https://doi.org/10.1145/3594805.3607136\">10.1145/3594805.3607136</a>","chicago":"Prager, Raphael Patrick, Konstantin Dietrich, Lennart Schneider, Lennart Schäpermeier, Bernd Bischl, Pascal Kerschke, Heike Trautmann, and Olaf Mersmann. “Neural Networks as Black-Box Benchmark Functions Optimized for Exploratory Landscape Features.” In <i>Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, 129–139. FOGA ’23. New York, NY, USA: Association for Computing Machinery, 2023. <a href=\"https://doi.org/10.1145/3594805.3607136\">https://doi.org/10.1145/3594805.3607136</a>.","ieee":"R. P. Prager <i>et al.</i>, “Neural Networks as Black-Box Benchmark Functions Optimized for Exploratory Landscape Features,” in <i>Proceedings of the 17th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, 2023, pp. 129–139, doi: <a href=\"https://doi.org/10.1145/3594805.3607136\">10.1145/3594805.3607136</a>."},"page":"129–139","publication_identifier":{"isbn":["9798400702020"]},"title":"Neural Networks as Black-Box Benchmark Functions Optimized for Exploratory Landscape Features","doi":"10.1145/3594805.3607136","date_updated":"2023-10-16T12:33:02Z","publisher":"Association for Computing Machinery","date_created":"2023-09-27T15:43:17Z","author":[{"first_name":"Raphael Patrick","last_name":"Prager","full_name":"Prager, Raphael Patrick"},{"first_name":"Konstantin","last_name":"Dietrich","full_name":"Dietrich, Konstantin"},{"first_name":"Lennart","last_name":"Schneider","full_name":"Schneider, Lennart"},{"full_name":"Schäpermeier, Lennart","last_name":"Schäpermeier","first_name":"Lennart"},{"last_name":"Bischl","full_name":"Bischl, Bernd","first_name":"Bernd"},{"last_name":"Kerschke","full_name":"Kerschke, Pascal","first_name":"Pascal"},{"id":"100740","full_name":"Trautmann, Heike","last_name":"Trautmann","orcid":"0000-0002-9788-8282","first_name":"Heike"},{"last_name":"Mersmann","full_name":"Mersmann, Olaf","first_name":"Olaf"}]},{"type":"conference","status":"public","editor":[{"last_name":"Rudolph","full_name":"Rudolph, Günter","first_name":"Günter"},{"first_name":"Anna V.","last_name":"Kononova","full_name":"Kononova, Anna V."},{"first_name":"Hernán","full_name":"Aguirre, Hernán","last_name":"Aguirre"},{"full_name":"Kerschke, Pascal","last_name":"Kerschke","first_name":"Pascal"},{"last_name":"Ochoa","full_name":"Ochoa, Gabriela","first_name":"Gabriela"},{"last_name":"Tusar","full_name":"Tusar, Tea","first_name":"Tea"}],"department":[{"_id":"819"}],"series_title":"Lecture Notes in Computer Science","user_id":"102979","_id":"48882","extern":"1","publication_identifier":{"isbn":["978-3-031-14714-2"]},"page":"192–206","citation":{"apa":"Heins, J., Rook, J., Schäpermeier, L., Kerschke, P., Bossek, J., &#38; Trautmann, H. (2022). BBE: Basin-Based Evaluation of Multimodal Multi-objective Optimization Problems. In G. Rudolph, A. V. Kononova, H. Aguirre, P. Kerschke, G. Ochoa, &#38; T. Tusar (Eds.), <i>Parallel Problem Solving from Nature (PPSN XVII)</i> (pp. 192–206). Springer International Publishing. <a href=\"https://doi.org/10.1007/978-3-031-14714-2_14\">https://doi.org/10.1007/978-3-031-14714-2_14</a>","short":"J. Heins, J. Rook, L. Schäpermeier, P. Kerschke, J. Bossek, H. Trautmann, in: G. Rudolph, A.V. Kononova, H. Aguirre, P. Kerschke, G. Ochoa, T. Tusar (Eds.), Parallel Problem Solving from Nature (PPSN XVII), Springer International Publishing, Cham, 2022, pp. 192–206.","mla":"Heins, Jonathan, et al. “BBE: Basin-Based Evaluation of Multimodal Multi-Objective Optimization Problems.” <i>Parallel Problem Solving from Nature (PPSN XVII)</i>, edited by Günter Rudolph et al., Springer International Publishing, 2022, pp. 192–206, doi:<a href=\"https://doi.org/10.1007/978-3-031-14714-2_14\">10.1007/978-3-031-14714-2_14</a>.","bibtex":"@inproceedings{Heins_Rook_Schäpermeier_Kerschke_Bossek_Trautmann_2022, place={Cham}, series={Lecture Notes in Computer Science}, title={BBE: Basin-Based Evaluation of Multimodal Multi-objective Optimization Problems}, DOI={<a href=\"https://doi.org/10.1007/978-3-031-14714-2_14\">10.1007/978-3-031-14714-2_14</a>}, booktitle={Parallel Problem Solving from Nature (PPSN XVII)}, publisher={Springer International Publishing}, author={Heins, Jonathan and Rook, Jeroen and Schäpermeier, Lennart and Kerschke, Pascal and Bossek, Jakob and Trautmann, Heike}, editor={Rudolph, Günter and Kononova, Anna V. and Aguirre, Hernán and Kerschke, Pascal and Ochoa, Gabriela and Tusar, Tea}, year={2022}, pages={192–206}, collection={Lecture Notes in Computer Science} }","chicago":"Heins, Jonathan, Jeroen Rook, Lennart Schäpermeier, Pascal Kerschke, Jakob Bossek, and Heike Trautmann. “BBE: Basin-Based Evaluation of Multimodal Multi-Objective Optimization Problems.” In <i>Parallel Problem Solving from Nature (PPSN XVII)</i>, edited by Günter Rudolph, Anna V. Kononova, Hernán Aguirre, Pascal Kerschke, Gabriela Ochoa, and Tea Tusar, 192–206. Lecture Notes in Computer Science. Cham: Springer International Publishing, 2022. <a href=\"https://doi.org/10.1007/978-3-031-14714-2_14\">https://doi.org/10.1007/978-3-031-14714-2_14</a>.","ieee":"J. Heins, J. Rook, L. Schäpermeier, P. Kerschke, J. Bossek, and H. Trautmann, “BBE: Basin-Based Evaluation of Multimodal Multi-objective Optimization Problems,” in <i>Parallel Problem Solving from Nature (PPSN XVII)</i>, 2022, pp. 192–206, doi: <a href=\"https://doi.org/10.1007/978-3-031-14714-2_14\">10.1007/978-3-031-14714-2_14</a>.","ama":"Heins J, Rook J, Schäpermeier L, Kerschke P, Bossek J, Trautmann H. BBE: Basin-Based Evaluation of Multimodal Multi-objective Optimization Problems. In: Rudolph G, Kononova AV, Aguirre H, Kerschke P, Ochoa G, Tusar T, eds. <i>Parallel Problem Solving from Nature (PPSN XVII)</i>. Lecture Notes in Computer Science. Springer International Publishing; 2022:192–206. doi:<a href=\"https://doi.org/10.1007/978-3-031-14714-2_14\">10.1007/978-3-031-14714-2_14</a>"},"place":"Cham","author":[{"full_name":"Heins, Jonathan","last_name":"Heins","first_name":"Jonathan"},{"full_name":"Rook, Jeroen","last_name":"Rook","first_name":"Jeroen"},{"full_name":"Schäpermeier, Lennart","last_name":"Schäpermeier","first_name":"Lennart"},{"first_name":"Pascal","full_name":"Kerschke, Pascal","last_name":"Kerschke"},{"id":"102979","full_name":"Bossek, Jakob","last_name":"Bossek","orcid":"0000-0002-4121-4668","first_name":"Jakob"},{"first_name":"Heike","last_name":"Trautmann","full_name":"Trautmann, Heike"}],"date_updated":"2023-12-13T10:47:50Z","doi":"10.1007/978-3-031-14714-2_14","publication":"Parallel Problem Solving from Nature (PPSN XVII)","abstract":[{"text":"In multimodal multi-objective optimization (MMMOO), the focus is not solely on convergence in objective space, but rather also on explicitly ensuring diversity in decision space. We illustrate why commonly used diversity measures are not entirely appropriate for this task and propose a sophisticated basin-based evaluation (BBE) method. Also, BBE variants are developed, capturing the anytime behavior of algorithms. The set of BBE measures is tested by means of an algorithm configuration study. We show that these new measures also transfer properties of the well-established hypervolume (HV) indicator to the domain of MMMOO, thus also accounting for objective space convergence. Moreover, we advance MMMOO research by providing insights into the multimodal performance of the considered algorithms. Specifically, algorithms exploiting local structures are shown to outperform classical evolutionary multi-objective optimizers regarding the BBE variants and respective trade-off with HV.","lang":"eng"}],"language":[{"iso":"eng"}],"keyword":["Anytime behavior","Benchmarking","Continuous optimization","Multi-objective optimization","Multimodality","Performance metric"],"year":"2022","date_created":"2023-11-14T15:58:58Z","publisher":"Springer International Publishing","title":"BBE: Basin-Based Evaluation of Multimodal Multi-objective Optimization Problems"},{"user_id":"15504","department":[{"_id":"34"},{"_id":"819"}],"_id":"46318","language":[{"iso":"eng"}],"keyword":["Multimodal optimization","Multi-objective continuous optimization","Landscape analysis","Visualization","Benchmarking","Theory","Algorithms"],"type":"journal_article","publication":"Computers & Operations Research","status":"public","abstract":[{"lang":"eng","text":"Multi-objective (MO) optimization, i.e., the simultaneous optimization of multiple conflicting objectives, is gaining more and more attention in various research areas, such as evolutionary computation, machine learning (e.g., (hyper-)parameter optimization), or logistics (e.g., vehicle routing). Many works in this domain mention the structural problem property of multimodality as a challenge from two classical perspectives: (1) finding all globally optimal solution sets, and (2) avoiding to get trapped in local optima. Interestingly, these streams seem to transfer many traditional concepts of single-objective (SO) optimization into claims, assumptions, or even terminology regarding the MO domain, but mostly neglect the understanding of the structural properties as well as the algorithmic search behavior on a problem’s landscape. However, some recent works counteract this trend, by investigating the fundamentals and characteristics of MO problems using new visualization techniques and gaining surprising insights. Using these visual insights, this work proposes a step towards a unified terminology to capture multimodality and locality in a broader way than it is usually done. This enables us to investigate current research activities in multimodal continuous MO optimization and to highlight new implications and promising research directions for the design of benchmark suites, the discovery of MO landscape features, the development of new MO (or even SO) optimization algorithms, and performance indicators. For all these topics, we provide a review of ideas and methods but also an outlook on future challenges, research potential and perspectives that result from recent developments."}],"author":[{"first_name":"Christian","last_name":"Grimme","full_name":"Grimme, Christian"},{"first_name":"Pascal","full_name":"Kerschke, Pascal","last_name":"Kerschke"},{"first_name":"Pelin","full_name":"Aspar, Pelin","last_name":"Aspar"},{"last_name":"Trautmann","orcid":"0000-0002-9788-8282","full_name":"Trautmann, Heike","id":"100740","first_name":"Heike"},{"full_name":"Preuss, Mike","last_name":"Preuss","first_name":"Mike"},{"last_name":"Deutz","full_name":"Deutz, André H.","first_name":"André H."},{"first_name":"Hao","full_name":"Wang, Hao","last_name":"Wang"},{"last_name":"Emmerich","full_name":"Emmerich, Michael","first_name":"Michael"}],"date_created":"2023-08-04T07:28:34Z","volume":136,"date_updated":"2023-10-16T12:58:42Z","doi":"https://doi.org/10.1016/j.cor.2021.105489","title":"Peeking beyond peaks: Challenges and research potentials of continuous multimodal multi-objective optimization","publication_identifier":{"issn":["0305-0548"]},"citation":{"ama":"Grimme C, Kerschke P, Aspar P, et al. Peeking beyond peaks: Challenges and research potentials of continuous multimodal multi-objective optimization. <i>Computers &#38; Operations Research</i>. 2021;136:105489. doi:<a href=\"https://doi.org/10.1016/j.cor.2021.105489\">https://doi.org/10.1016/j.cor.2021.105489</a>","chicago":"Grimme, Christian, Pascal Kerschke, Pelin Aspar, Heike Trautmann, Mike Preuss, André H. Deutz, Hao Wang, and Michael Emmerich. “Peeking beyond Peaks: Challenges and Research Potentials of Continuous Multimodal Multi-Objective Optimization.” <i>Computers &#38; Operations Research</i> 136 (2021): 105489. <a href=\"https://doi.org/10.1016/j.cor.2021.105489\">https://doi.org/10.1016/j.cor.2021.105489</a>.","ieee":"C. Grimme <i>et al.</i>, “Peeking beyond peaks: Challenges and research potentials of continuous multimodal multi-objective optimization,” <i>Computers &#38; Operations Research</i>, vol. 136, p. 105489, 2021, doi: <a href=\"https://doi.org/10.1016/j.cor.2021.105489\">https://doi.org/10.1016/j.cor.2021.105489</a>.","bibtex":"@article{Grimme_Kerschke_Aspar_Trautmann_Preuss_Deutz_Wang_Emmerich_2021, title={Peeking beyond peaks: Challenges and research potentials of continuous multimodal multi-objective optimization}, volume={136}, DOI={<a href=\"https://doi.org/10.1016/j.cor.2021.105489\">https://doi.org/10.1016/j.cor.2021.105489</a>}, journal={Computers &#38; Operations Research}, author={Grimme, Christian and Kerschke, Pascal and Aspar, Pelin and Trautmann, Heike and Preuss, Mike and Deutz, André H. and Wang, Hao and Emmerich, Michael}, year={2021}, pages={105489} }","mla":"Grimme, Christian, et al. “Peeking beyond Peaks: Challenges and Research Potentials of Continuous Multimodal Multi-Objective Optimization.” <i>Computers &#38; Operations Research</i>, vol. 136, 2021, p. 105489, doi:<a href=\"https://doi.org/10.1016/j.cor.2021.105489\">https://doi.org/10.1016/j.cor.2021.105489</a>.","short":"C. Grimme, P. Kerschke, P. Aspar, H. Trautmann, M. Preuss, A.H. Deutz, H. Wang, M. Emmerich, Computers &#38; Operations Research 136 (2021) 105489.","apa":"Grimme, C., Kerschke, P., Aspar, P., Trautmann, H., Preuss, M., Deutz, A. H., Wang, H., &#38; Emmerich, M. (2021). Peeking beyond peaks: Challenges and research potentials of continuous multimodal multi-objective optimization. <i>Computers &#38; Operations Research</i>, <i>136</i>, 105489. <a href=\"https://doi.org/10.1016/j.cor.2021.105489\">https://doi.org/10.1016/j.cor.2021.105489</a>"},"page":"105489","intvolume":"       136","year":"2021"},{"publication":"Semantic Web","file":[{"success":1,"relation":"main_file","content_type":"application/pdf","file_size":978478,"file_name":"swj2604.pdf","file_id":"50483","access_level":"closed","date_updated":"2024-01-13T11:35:53Z","creator":"uqudus","date_created":"2024-01-13T11:35:53Z"}],"abstract":[{"lang":"eng","text":"Finding a good query plan is key to the optimization of query runtime. This holds in particular for cost-based federation\r\nengines, which make use of cardinality estimations to achieve this goal. A number of studies compare SPARQL federation engines across different performance metrics, including query runtime, result set completeness and correctness, number of sources selected and number of requests sent. Albeit informative, these metrics are generic and unable to quantify and evaluate the accuracy of the cardinality estimators of cost-based federation engines. To thoroughly evaluate cost-based federation engines, the effect of estimated cardinality errors on the overall query runtime performance must be measured. In this paper, we address this challenge by presenting novel evaluation metrics targeted at a fine-grained benchmarking of cost-based federated SPARQL query engines. We evaluate five cost-based federated SPARQL query engines using existing as well as novel evaluation metrics by using LargeRDFBench queries. Our results provide a detailed analysis of the experimental outcomes that reveal novel insights, useful for the development of future cost-based federated SPARQL query processing engines."}],"language":[{"iso":"eng"}],"keyword":["SPARQL","benchmarking","cost-based","cost-free","federated","querying"],"ddc":["000"],"issue":"6","year":"2021","date_created":"2021-10-01T06:52:52Z","publisher":"ISO Press","title":"An Empirical Evaluation of Cost-based Federated SPARQL Query Processing Engines","type":"journal_article","status":"public","department":[{"_id":"574"}],"user_id":"83392","_id":"25212","project":[{"_id":"410","name":"KnowGraphs: KnowGraphs: Knowledge Graphs at Scale"}],"file_date_updated":"2024-01-13T11:35:53Z","article_type":"original","publication_identifier":{"issn":["2210-4968"]},"has_accepted_license":"1","publication_status":"accepted","page":"843-868","intvolume":"        12","citation":{"ama":"Qudus U, Saleem M, Ngonga Ngomo A-C, Lee Y-K. An Empirical Evaluation of Cost-based Federated SPARQL Query Processing Engines. <i>Semantic Web</i>. 12(6):843-868. doi:<a href=\"https://doi.org/10.3233/SW-200420\">10.3233/SW-200420</a>","chicago":"Qudus, Umair, Muhammad Saleem, Axel-Cyrille Ngonga Ngomo, and Young-Koo Lee. “An Empirical Evaluation of Cost-Based Federated SPARQL Query Processing Engines.” <i>Semantic Web</i> 12, no. 6 (n.d.): 843–68. <a href=\"https://doi.org/10.3233/SW-200420\">https://doi.org/10.3233/SW-200420</a>.","ieee":"U. Qudus, M. Saleem, A.-C. Ngonga Ngomo, and Y.-K. Lee, “An Empirical Evaluation of Cost-based Federated SPARQL Query Processing Engines,” <i>Semantic Web</i>, vol. 12, no. 6, pp. 843–868, doi: <a href=\"https://doi.org/10.3233/SW-200420\">10.3233/SW-200420</a>.","apa":"Qudus, U., Saleem, M., Ngonga Ngomo, A.-C., &#38; Lee, Y.-K. (n.d.). An Empirical Evaluation of Cost-based Federated SPARQL Query Processing Engines. <i>Semantic Web</i>, <i>12</i>(6), 843–868. <a href=\"https://doi.org/10.3233/SW-200420\">https://doi.org/10.3233/SW-200420</a>","mla":"Qudus, Umair, et al. “An Empirical Evaluation of Cost-Based Federated SPARQL Query Processing Engines.” <i>Semantic Web</i>, vol. 12, no. 6, ISO Press, pp. 843–68, doi:<a href=\"https://doi.org/10.3233/SW-200420\">10.3233/SW-200420</a>.","bibtex":"@article{Qudus_Saleem_Ngonga Ngomo_Lee, title={An Empirical Evaluation of Cost-based Federated SPARQL Query Processing Engines}, volume={12}, DOI={<a href=\"https://doi.org/10.3233/SW-200420\">10.3233/SW-200420</a>}, number={6}, journal={Semantic Web}, publisher={ISO Press}, author={Qudus, Umair and Saleem, Muhammad and Ngonga Ngomo, Axel-Cyrille and Lee, Young-Koo}, pages={843–868} }","short":"U. Qudus, M. Saleem, A.-C. Ngonga Ngomo, Y.-K. Lee, Semantic Web 12 (n.d.) 843–868."},"volume":12,"author":[{"first_name":"Umair","id":"83392","full_name":"Qudus, Umair","orcid":"0000-0001-6714-8729","last_name":"Qudus"},{"first_name":"Muhammad","full_name":"Saleem, Muhammad","last_name":"Saleem"},{"first_name":"Axel-Cyrille","id":"65716","full_name":"Ngonga Ngomo, Axel-Cyrille","last_name":"Ngonga Ngomo"},{"last_name":"Lee","full_name":"Lee, Young-Koo","first_name":"Young-Koo"}],"date_updated":"2025-09-11T09:50:14Z","doi":"10.3233/SW-200420"},{"year":"2020","quality_controlled":"1","title":"Evaluating FPGA Accelerator Performance with a Parameterized OpenCL Adaptation of Selected Benchmarks of the HPCChallenge Benchmark Suite","date_created":"2021-04-16T10:17:22Z","abstract":[{"lang":"eng","text":"FPGAs have found increasing adoption in data center applications since a new generation of high-level tools have become available which noticeably reduce development time for FPGA accelerators and still provide high-quality results. There is, however, no high-level benchmark suite available, which specifically enables a comparison of FPGA architectures, programming tools, and libraries for HPC applications. To fill this gap, we have developed an OpenCL-based open-source implementation of the HPCC benchmark suite for Xilinx and Intel FPGAs. This benchmark can serve to analyze the current capabilities of FPGA devices, cards, and development tool flows, track progress over time, and point out specific difficulties for FPGA acceleration in the HPC domain. Additionally, the benchmark documents proven performance optimization patterns. We will continue optimizing and porting the benchmark for new generations of FPGAs and design tools and encourage active participation to create a valuable tool for the community. To fill this gap, we have developed an OpenCL-based open-source implementation of the HPCC benchmark suite for Xilinx and Intel FPGAs. This benchmark can serve to analyze the current capabilities of FPGA devices, cards, and development tool flows, track progress over time, and point out specific difficulties for FPGA acceleration in the HPC domain. Additionally, the benchmark documents proven performance optimization patterns. We will continue optimizing and porting the benchmark for new generations of FPGAs and design tools and encourage active participation to create a valuable tool for the community."}],"publication":"2020 IEEE/ACM International Workshop on Heterogeneous High-performance Reconfigurable Computing (H2RC)","language":[{"iso":"eng"}],"keyword":["FPGA","OpenCL","High Level Synthesis","HPC benchmarking"],"citation":{"bibtex":"@inproceedings{Meyer_Kenter_Plessl_2020, title={Evaluating FPGA Accelerator Performance with a Parameterized OpenCL Adaptation of Selected Benchmarks of the HPCChallenge Benchmark Suite}, DOI={<a href=\"https://doi.org/10.1109/h2rc51942.2020.00007\">10.1109/h2rc51942.2020.00007</a>}, booktitle={2020 IEEE/ACM International Workshop on Heterogeneous High-performance Reconfigurable Computing (H2RC)}, author={Meyer, Marius and Kenter, Tobias and Plessl, Christian}, year={2020} }","short":"M. Meyer, T. Kenter, C. Plessl, in: 2020 IEEE/ACM International Workshop on Heterogeneous High-Performance Reconfigurable Computing (H2RC), 2020.","mla":"Meyer, Marius, et al. “Evaluating FPGA Accelerator Performance with a Parameterized OpenCL Adaptation of Selected Benchmarks of the HPCChallenge Benchmark Suite.” <i>2020 IEEE/ACM International Workshop on Heterogeneous High-Performance Reconfigurable Computing (H2RC)</i>, 2020, doi:<a href=\"https://doi.org/10.1109/h2rc51942.2020.00007\">10.1109/h2rc51942.2020.00007</a>.","apa":"Meyer, M., Kenter, T., &#38; Plessl, C. (2020). Evaluating FPGA Accelerator Performance with a Parameterized OpenCL Adaptation of Selected Benchmarks of the HPCChallenge Benchmark Suite. <i>2020 IEEE/ACM International Workshop on Heterogeneous High-Performance Reconfigurable Computing (H2RC)</i>. <a href=\"https://doi.org/10.1109/h2rc51942.2020.00007\">https://doi.org/10.1109/h2rc51942.2020.00007</a>","ama":"Meyer M, Kenter T, Plessl C. Evaluating FPGA Accelerator Performance with a Parameterized OpenCL Adaptation of Selected Benchmarks of the HPCChallenge Benchmark Suite. In: <i>2020 IEEE/ACM International Workshop on Heterogeneous High-Performance Reconfigurable Computing (H2RC)</i>. ; 2020. doi:<a href=\"https://doi.org/10.1109/h2rc51942.2020.00007\">10.1109/h2rc51942.2020.00007</a>","ieee":"M. Meyer, T. Kenter, and C. Plessl, “Evaluating FPGA Accelerator Performance with a Parameterized OpenCL Adaptation of Selected Benchmarks of the HPCChallenge Benchmark Suite,” 2020, doi: <a href=\"https://doi.org/10.1109/h2rc51942.2020.00007\">10.1109/h2rc51942.2020.00007</a>.","chicago":"Meyer, Marius, Tobias Kenter, and Christian Plessl. “Evaluating FPGA Accelerator Performance with a Parameterized OpenCL Adaptation of Selected Benchmarks of the HPCChallenge Benchmark Suite.” In <i>2020 IEEE/ACM International Workshop on Heterogeneous High-Performance Reconfigurable Computing (H2RC)</i>, 2020. <a href=\"https://doi.org/10.1109/h2rc51942.2020.00007\">https://doi.org/10.1109/h2rc51942.2020.00007</a>."},"related_material":{"link":[{"url":"https://github.com/pc2/HPCC_FPGA","relation":"supplementary_material","description":"Official repository of the benchmark suite on GitHub"}]},"publication_status":"published","publication_identifier":{"isbn":["9781665415927"]},"main_file_link":[{"url":"https://ieeexplore.ieee.org/document/9306963"}],"doi":"10.1109/h2rc51942.2020.00007","author":[{"id":"40778","full_name":"Meyer, Marius","last_name":"Meyer","first_name":"Marius"},{"id":"3145","full_name":"Kenter, Tobias","last_name":"Kenter","first_name":"Tobias"},{"first_name":"Christian","full_name":"Plessl, Christian","id":"16153","orcid":"0000-0001-5728-9982","last_name":"Plessl"}],"date_updated":"2023-09-26T11:42:53Z","status":"public","type":"conference","user_id":"15278","department":[{"_id":"27"},{"_id":"518"}],"project":[{"_id":"52","name":"Computing Resources Provided by the Paderborn Center for Parallel Computing"}],"_id":"21632"},{"extern":"1","department":[{"_id":"819"}],"user_id":"102979","series_title":"FOGA ’19","_id":"48842","status":"public","type":"conference","doi":"10.1145/3299904.3340307","author":[{"orcid":"0000-0002-4121-4668","last_name":"Bossek","full_name":"Bossek, Jakob","id":"102979","first_name":"Jakob"},{"full_name":"Kerschke, Pascal","last_name":"Kerschke","first_name":"Pascal"},{"first_name":"Aneta","full_name":"Neumann, Aneta","last_name":"Neumann"},{"first_name":"Markus","full_name":"Wagner, Markus","last_name":"Wagner"},{"first_name":"Frank","full_name":"Neumann, Frank","last_name":"Neumann"},{"first_name":"Heike","full_name":"Trautmann, Heike","last_name":"Trautmann"}],"date_updated":"2023-12-13T10:42:57Z","page":"58–71","citation":{"ama":"Bossek J, Kerschke P, Neumann A, Wagner M, Neumann F, Trautmann H. Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators. In: <i>Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>. FOGA ’19. Association for Computing Machinery; 2019:58–71. doi:<a href=\"https://doi.org/10.1145/3299904.3340307\">10.1145/3299904.3340307</a>","chicago":"Bossek, Jakob, Pascal Kerschke, Aneta Neumann, Markus Wagner, Frank Neumann, and Heike Trautmann. “Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators.” In <i>Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, 58–71. FOGA ’19. New York, NY, USA: Association for Computing Machinery, 2019. <a href=\"https://doi.org/10.1145/3299904.3340307\">https://doi.org/10.1145/3299904.3340307</a>.","ieee":"J. Bossek, P. Kerschke, A. Neumann, M. Wagner, F. Neumann, and H. Trautmann, “Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators,” in <i>Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, 2019, pp. 58–71, doi: <a href=\"https://doi.org/10.1145/3299904.3340307\">10.1145/3299904.3340307</a>.","mla":"Bossek, Jakob, et al. “Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators.” <i>Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, Association for Computing Machinery, 2019, pp. 58–71, doi:<a href=\"https://doi.org/10.1145/3299904.3340307\">10.1145/3299904.3340307</a>.","bibtex":"@inproceedings{Bossek_Kerschke_Neumann_Wagner_Neumann_Trautmann_2019, place={New York, NY, USA}, series={FOGA ’19}, title={Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators}, DOI={<a href=\"https://doi.org/10.1145/3299904.3340307\">10.1145/3299904.3340307</a>}, booktitle={Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms}, publisher={Association for Computing Machinery}, author={Bossek, Jakob and Kerschke, Pascal and Neumann, Aneta and Wagner, Markus and Neumann, Frank and Trautmann, Heike}, year={2019}, pages={58–71}, collection={FOGA ’19} }","short":"J. Bossek, P. Kerschke, A. Neumann, M. Wagner, F. Neumann, H. Trautmann, in: Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms, Association for Computing Machinery, New York, NY, USA, 2019, pp. 58–71.","apa":"Bossek, J., Kerschke, P., Neumann, A., Wagner, M., Neumann, F., &#38; Trautmann, H. (2019). Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators. <i>Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms</i>, 58–71. <a href=\"https://doi.org/10.1145/3299904.3340307\">https://doi.org/10.1145/3299904.3340307</a>"},"place":"New York, NY, USA","publication_identifier":{"isbn":["978-1-4503-6254-2"]},"publication_status":"published","language":[{"iso":"eng"}],"keyword":["benchmarking","instance features","optimization","problem generation","traveling salesperson problem"],"abstract":[{"text":"Evolutionary algorithms have successfully been applied to evolve problem instances that exhibit a significant difference in performance for a given algorithm or a pair of algorithms inter alia for the Traveling Salesperson Problem (TSP). Creating a large variety of instances is crucial for successful applications in the blooming field of algorithm selection. In this paper, we introduce new and creative mutation operators for evolving instances of the TSP. We show that adopting those operators in an evolutionary algorithm allows for the generation of benchmark sets with highly desirable properties: (1) novelty by clear visual distinction to established benchmark sets in the field, (2) visual and quantitative diversity in the space of TSP problem characteristics, and (3) significant performance differences with respect to the restart versions of heuristic state-of-the-art TSP solvers EAX and LKH. The important aspect of diversity is addressed and achieved solely by the proposed mutation operators and not enforced by explicit diversity preservation.","lang":"eng"}],"publication":"Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms","title":"Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators","date_created":"2023-11-14T15:58:52Z","publisher":"Association for Computing Machinery","year":"2019"},{"language":[{"iso":"eng"}],"keyword":["machine learning","exploratory landscape analysis","fitness landscape","benchmarking","evolutionary optimization","bbob test set","algorithm selection"],"department":[{"_id":"34"},{"_id":"819"}],"series_title":"GECCO ’12","user_id":"15504","_id":"46396","status":"public","abstract":[{"text":"The steady supply of new optimization methods makes the algorithm selection problem (ASP) an increasingly pressing and challenging task, specially for real-world black-box optimization problems. The introduced approach considers the ASP as a cost-sensitive classification task which is based on Exploratory Landscape Analysis. Low-level features gathered by systematic sampling of the function on the feasible set are used to predict a well-performing algorithm out of a given portfolio. Example-specific label costs are defined by the expected runtime of each candidate algorithm. We use one-sided support vector regression to solve this learning problem. The approach is illustrated by means of the optimization problems and algorithms of the BBOB’09/10 workshop.","lang":"eng"}],"publication":"Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation","type":"conference","doi":"10.1145/2330163.2330209","title":"Algorithm Selection Based on Exploratory Landscape Analysis and Cost-Sensitive Learning","date_created":"2023-08-04T15:51:56Z","author":[{"first_name":"Bernd","last_name":"Bischl","full_name":"Bischl, Bernd"},{"first_name":"Olaf","full_name":"Mersmann, Olaf","last_name":"Mersmann"},{"orcid":"0000-0002-9788-8282","last_name":"Trautmann","full_name":"Trautmann, Heike","id":"100740","first_name":"Heike"},{"first_name":"Mike","full_name":"Preuß, Mike","last_name":"Preuß"}],"publisher":"Association for Computing Machinery","date_updated":"2023-10-16T13:48:48Z","page":"313–320","citation":{"short":"B. Bischl, O. Mersmann, H. Trautmann, M. Preuß, in: Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation, Association for Computing Machinery, New York, NY, USA, 2012, pp. 313–320.","bibtex":"@inproceedings{Bischl_Mersmann_Trautmann_Preuß_2012, place={New York, NY, USA}, series={GECCO ’12}, title={Algorithm Selection Based on Exploratory Landscape Analysis and Cost-Sensitive Learning}, DOI={<a href=\"https://doi.org/10.1145/2330163.2330209\">10.1145/2330163.2330209</a>}, booktitle={Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation}, publisher={Association for Computing Machinery}, author={Bischl, Bernd and Mersmann, Olaf and Trautmann, Heike and Preuß, Mike}, year={2012}, pages={313–320}, collection={GECCO ’12} }","mla":"Bischl, Bernd, et al. “Algorithm Selection Based on Exploratory Landscape Analysis and Cost-Sensitive Learning.” <i>Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation</i>, Association for Computing Machinery, 2012, pp. 313–320, doi:<a href=\"https://doi.org/10.1145/2330163.2330209\">10.1145/2330163.2330209</a>.","apa":"Bischl, B., Mersmann, O., Trautmann, H., &#38; Preuß, M. (2012). Algorithm Selection Based on Exploratory Landscape Analysis and Cost-Sensitive Learning. <i>Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation</i>, 313–320. <a href=\"https://doi.org/10.1145/2330163.2330209\">https://doi.org/10.1145/2330163.2330209</a>","ieee":"B. Bischl, O. Mersmann, H. Trautmann, and M. Preuß, “Algorithm Selection Based on Exploratory Landscape Analysis and Cost-Sensitive Learning,” in <i>Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation</i>, 2012, pp. 313–320, doi: <a href=\"https://doi.org/10.1145/2330163.2330209\">10.1145/2330163.2330209</a>.","chicago":"Bischl, Bernd, Olaf Mersmann, Heike Trautmann, and Mike Preuß. “Algorithm Selection Based on Exploratory Landscape Analysis and Cost-Sensitive Learning.” In <i>Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation</i>, 313–320. GECCO ’12. New York, NY, USA: Association for Computing Machinery, 2012. <a href=\"https://doi.org/10.1145/2330163.2330209\">https://doi.org/10.1145/2330163.2330209</a>.","ama":"Bischl B, Mersmann O, Trautmann H, Preuß M. Algorithm Selection Based on Exploratory Landscape Analysis and Cost-Sensitive Learning. In: <i>Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation</i>. GECCO ’12. Association for Computing Machinery; 2012:313–320. doi:<a href=\"https://doi.org/10.1145/2330163.2330209\">10.1145/2330163.2330209</a>"},"place":"New York, NY, USA","year":"2012","publication_identifier":{"isbn":["9781450311779"]}},{"language":[{"iso":"eng"}],"keyword":["exploratory landscape analysis","evolutionary optimization","fitness landscape","benchmarking","BBOB test set"],"series_title":"GECCO ’11","user_id":"15504","department":[{"_id":"34"},{"_id":"819"}],"_id":"46401","status":"public","abstract":[{"lang":"eng","text":"Exploratory Landscape Analysis subsumes a number of techniques employed to obtain knowledge about the properties of an unknown optimization problem, especially insofar as these properties are important for the performance of optimization algorithms. Where in a first attempt, one could rely on high-level features designed by experts, we approach the problem from a different angle here, namely by using relatively cheap low-level computer generated features. Interestingly, very few features are needed to separate the BBOB problem groups and also for relating a problem to high-level, expert designed features, paving the way for automatic algorithm selection."}],"type":"conference","publication":"Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation","doi":"10.1145/2001576.2001690","title":"Exploratory Landscape Analysis","date_created":"2023-08-04T15:58:22Z","author":[{"first_name":"Olaf","last_name":"Mersmann","full_name":"Mersmann, Olaf"},{"first_name":"Bernd","full_name":"Bischl, Bernd","last_name":"Bischl"},{"first_name":"Heike","last_name":"Trautmann","orcid":"0000-0002-9788-8282","full_name":"Trautmann, Heike","id":"100740"},{"full_name":"Preuss, Mike","last_name":"Preuss","first_name":"Mike"},{"last_name":"Weihs","full_name":"Weihs, Claus","first_name":"Claus"},{"full_name":"Rudolph, Günter","last_name":"Rudolph","first_name":"Günter"}],"publisher":"Association for Computing Machinery","date_updated":"2023-10-16T13:54:34Z","citation":{"ieee":"O. Mersmann, B. Bischl, H. Trautmann, M. Preuss, C. Weihs, and G. Rudolph, “Exploratory Landscape Analysis,” in <i>Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation</i>, 2011, pp. 829–836, doi: <a href=\"https://doi.org/10.1145/2001576.2001690\">10.1145/2001576.2001690</a>.","chicago":"Mersmann, Olaf, Bernd Bischl, Heike Trautmann, Mike Preuss, Claus Weihs, and Günter Rudolph. “Exploratory Landscape Analysis.” In <i>Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation</i>, 829–836. GECCO ’11. New York, NY, USA: Association for Computing Machinery, 2011. <a href=\"https://doi.org/10.1145/2001576.2001690\">https://doi.org/10.1145/2001576.2001690</a>.","ama":"Mersmann O, Bischl B, Trautmann H, Preuss M, Weihs C, Rudolph G. Exploratory Landscape Analysis. In: <i>Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation</i>. GECCO ’11. Association for Computing Machinery; 2011:829–836. doi:<a href=\"https://doi.org/10.1145/2001576.2001690\">10.1145/2001576.2001690</a>","short":"O. Mersmann, B. Bischl, H. Trautmann, M. Preuss, C. Weihs, G. Rudolph, in: Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation, Association for Computing Machinery, New York, NY, USA, 2011, pp. 829–836.","bibtex":"@inproceedings{Mersmann_Bischl_Trautmann_Preuss_Weihs_Rudolph_2011, place={New York, NY, USA}, series={GECCO ’11}, title={Exploratory Landscape Analysis}, DOI={<a href=\"https://doi.org/10.1145/2001576.2001690\">10.1145/2001576.2001690</a>}, booktitle={Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation}, publisher={Association for Computing Machinery}, author={Mersmann, Olaf and Bischl, Bernd and Trautmann, Heike and Preuss, Mike and Weihs, Claus and Rudolph, Günter}, year={2011}, pages={829–836}, collection={GECCO ’11} }","mla":"Mersmann, Olaf, et al. “Exploratory Landscape Analysis.” <i>Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation</i>, Association for Computing Machinery, 2011, pp. 829–836, doi:<a href=\"https://doi.org/10.1145/2001576.2001690\">10.1145/2001576.2001690</a>.","apa":"Mersmann, O., Bischl, B., Trautmann, H., Preuss, M., Weihs, C., &#38; Rudolph, G. (2011). Exploratory Landscape Analysis. <i>Proceedings of the 13th Annual Conference on Genetic and Evolutionary Computation</i>, 829–836. <a href=\"https://doi.org/10.1145/2001576.2001690\">https://doi.org/10.1145/2001576.2001690</a>"},"page":"829–836","year":"2011","place":"New York, NY, USA","publication_identifier":{"isbn":["9781450305570"]}},{"keyword":["benchmarking","multidimensional scaling","consensus ranking","evolutionary optimization","BBOB test set"],"language":[{"iso":"eng"}],"_id":"46405","user_id":"15504","series_title":"PPSN’10","department":[{"_id":"34"},{"_id":"819"}],"abstract":[{"lang":"eng","text":"We present methods to answer two basic questions that arise when benchmarking optimization algorithms. The first one is: which algorithm is the ’best’ one? and the second one: which algorithm should I use for my real world problem? Both are connected and neither is easy to answer. We present methods which can be used to analyse the raw data of a benchmark experiment and derive some insight regarding the answers to these questions. We employ the presented methods to analyse the BBOB’09 benchmark results and present some initial findings."}],"status":"public","type":"conference","publication":"Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I","title":"Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis","date_updated":"2023-10-16T13:55:43Z","publisher":"Springer-Verlag","author":[{"last_name":"Mersmann","full_name":"Mersmann, Olaf","first_name":"Olaf"},{"first_name":"Mike","full_name":"Preuss, Mike","last_name":"Preuss"},{"id":"100740","full_name":"Trautmann, Heike","last_name":"Trautmann","orcid":"0000-0002-9788-8282","first_name":"Heike"}],"date_created":"2023-08-04T16:02:28Z","year":"2010","place":"Berlin, Heidelberg","citation":{"ama":"Mersmann O, Preuss M, Trautmann H. Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis. In: <i>Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I</i>. PPSN’10. Springer-Verlag; 2010:73–82.","ieee":"O. Mersmann, M. Preuss, and H. Trautmann, “Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis,” in <i>Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I</i>, 2010, pp. 73–82.","chicago":"Mersmann, Olaf, Mike Preuss, and Heike Trautmann. “Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis.” In <i>Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I</i>, 73–82. PPSN’10. Berlin, Heidelberg: Springer-Verlag, 2010.","apa":"Mersmann, O., Preuss, M., &#38; Trautmann, H. (2010). Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis. <i>Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I</i>, 73–82.","short":"O. Mersmann, M. Preuss, H. Trautmann, in: Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I, Springer-Verlag, Berlin, Heidelberg, 2010, pp. 73–82.","mla":"Mersmann, Olaf, et al. “Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis.” <i>Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I</i>, Springer-Verlag, 2010, pp. 73–82.","bibtex":"@inproceedings{Mersmann_Preuss_Trautmann_2010, place={Berlin, Heidelberg}, series={PPSN’10}, title={Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis}, booktitle={Proceedings of the 11th International Conference on Parallel Problem Solving from Nature: Part I}, publisher={Springer-Verlag}, author={Mersmann, Olaf and Preuss, Mike and Trautmann, Heike}, year={2010}, pages={73–82}, collection={PPSN’10} }"},"page":"73–82","publication_identifier":{"isbn":["3642158439"]}}]