@inproceedings{48852, abstract = {{The Traveling Salesperson Problem (TSP) is one of the best-known combinatorial optimisation problems. However, many real-world problems are composed of several interacting components. The Traveling Thief Problem (TTP) addresses such interactions by combining two combinatorial optimisation problems, namely the TSP and the Knapsack Problem (KP). Recently, a new problem called the node weight dependent Traveling Salesperson Problem (W-TSP) has been introduced where nodes have weights that influence the cost of the tour. In this paper, we compare W-TSP and TTP. We investigate the structure of the optimised tours for W-TSP and TTP and the impact of using each others fitness function. Our experimental results suggest (1) that the W-TSP often can be solved better using the TTP fitness function and (2) final W-TSP and TTP solutions show different distributions when compared with optimal TSP or weighted greedy solutions.}}, author = {{Bossek, Jakob and Neumann, Aneta and Neumann, Frank}}, booktitle = {{Parallel Problem Solving from Nature (PPSN XVI)}}, isbn = {{978-3-030-58111-4}}, keywords = {{Evolutionary algorithms, Node weight dependent TSP, Traveling Thief Problem}}, pages = {{346–359}}, publisher = {{Springer-Verlag}}, title = {{{Optimising Tours for the Weighted Traveling Salesperson Problem and the Traveling Thief Problem: A Structural Comparison of Solutions}}}, doi = {{10.1007/978-3-030-58112-1_24}}, year = {{2020}}, } @inproceedings{48846, abstract = {{We consider a dynamic bi-objective vehicle routing problem, where a subset of customers ask for service over time. Therein, the distance traveled by a single vehicle and the number of unserved dynamic requests is minimized by a dynamic evolutionary multi-objective algorithm (DEMOA), which operates on discrete time windows (eras). A decision is made at each era by a decision-maker, thus any decision depends on irreversible decisions made in foregoing eras. To understand effects of sequences of decision-making and interactions/dependencies between decisions made, we conduct a series of experiments. More precisely, we fix a set of decision-maker preferences D and the number of eras n{$<$}inf{$>$}t{$<$}/inf{$>$} and analyze all $|D|\^{n_t}$ combinations of decision-maker options. We find that for random uniform instances (a) the final selected solutions mainly depend on the final decision and not on the decision history, (b) solutions are quite robust with respect to the number of unvisited dynamic customers, and (c) solutions of the dynamic approach can even dominate solutions obtained by a clairvoyant EMOA. In contrast, for instances with clustered customers, we observe a strong dependency on decision-making history as well as more variance in solution diversity.}}, author = {{Bossek, Jakob and Grimme, Christian and Rudolph, Günter and Trautmann, Heike}}, booktitle = {{2020 IEEE Congress on Evolutionary Computation (CEC)}}, pages = {{1–8}}, publisher = {{IEEE Press}}, title = {{{Towards Decision Support in Dynamic Bi-Objective Vehicle Routing}}}, doi = {{10.1109/CEC48606.2020.9185778}}, year = {{2020}}, } @inproceedings{48879, abstract = {{Evolving diverse sets of high quality solutions has gained increasing interest in the evolutionary computation literature in recent years. With this paper, we contribute to this area of research by examining evolutionary diversity optimisation approaches for the classical Traveling Salesperson Problem (TSP). We study the impact of using different diversity measures for a given set of tours and the ability of evolutionary algorithms to obtain a diverse set of high quality solutions when adopting these measures. Our studies show that a large variety of diverse high quality tours can be achieved by using our approaches. Furthermore, we compare our approaches in terms of theoretical properties and the final set of tours obtained by the evolutionary diversity optimisation algorithm.}}, author = {{Do, Anh Viet and Bossek, Jakob and Neumann, Aneta and Neumann, Frank}}, booktitle = {{Proceedings of the Genetic and Evolutionary Computation Conference}}, isbn = {{978-1-4503-7128-5}}, keywords = {{diversity maximisation, evolutionary algorithms, travelling salesperson problem}}, pages = {{681–689}}, publisher = {{Association for Computing Machinery}}, title = {{{Evolving Diverse Sets of Tours for the Travelling Salesperson Problem}}}, doi = {{10.1145/3377930.3389844}}, year = {{2020}}, } @inproceedings{48895, abstract = {{Evolutionary algorithms (EAs) are general-purpose problem solvers that usually perform an unbiased search. This is reasonable and desirable in a black-box scenario. For combinatorial optimization problems, often more knowledge about the structure of optimal solutions is given, which can be leveraged by means of biased search operators. We consider the Minimum Spanning Tree (MST) problem in a single- and multi-objective version, and introduce a biased mutation, which puts more emphasis on the selection of edges of low rank in terms of low domination number. We present example graphs where the biased mutation can significantly speed up the expected runtime until (Pareto-)optimal solutions are found. On the other hand, we demonstrate that bias can lead to exponential runtime if "heavy" edges are necessarily part of an optimal solution. However, on general graphs in the single-objective setting, we show that a combined mutation operator which decides for unbiased or biased edge selection in each step with equal probability exhibits a polynomial upper bound - as unbiased mutation - in the worst case and benefits from bias if the circumstances are favorable.}}, author = {{Roostapour, Vahid and Bossek, Jakob and Neumann, Frank}}, booktitle = {{Proceedings of the 2020 Genetic and Evolutionary Computation Conference}}, isbn = {{978-1-4503-7128-5}}, keywords = {{biased mutation, evolutionary algorithms, minimum spanning tree problem, runtime analysis}}, pages = {{551–559}}, publisher = {{Association for Computing Machinery}}, title = {{{Runtime Analysis of Evolutionary Algorithms with Biased Mutation for the Multi-Objective Minimum Spanning Tree Problem}}}, doi = {{10.1145/3377930.3390168}}, year = {{2020}}, } @inproceedings{48897, abstract = {{In this work we focus on the well-known Euclidean Traveling Salesperson Problem (TSP) and two highly competitive inexact heuristic TSP solvers, EAX and LKH, in the context of per-instance algorithm selection (AS). We evolve instances with nodes where the solvers show strongly different performance profiles. These instances serve as a basis for an exploratory study on the identification of well-discriminating problem characteristics (features). Our results in a nutshell: we show that even though (1) promising features exist, (2) these are in line with previous results from the literature, and (3) models trained with these features are more accurate than models adopting sophisticated feature selection methods, the advantage is not close to the virtual best solver in terms of penalized average runtime and so is the performance gain over the single best solver. However, we show that a feature-free deep neural network based approach solely based on visual representation of the instances already matches classical AS model results and thus shows huge potential for future studies.}}, author = {{Seiler, Moritz and Pohl, Janina and Bossek, Jakob and Kerschke, Pascal and Trautmann, Heike}}, booktitle = {{Parallel Problem Solving from {Nature} (PPSN XVI)}}, isbn = {{978-3-030-58111-4}}, keywords = {{Automated algorithm selection, Deep learning, Feature-based approaches, Traveling Salesperson Problem}}, pages = {{48–64}}, publisher = {{Springer-Verlag}}, title = {{{Deep Learning as a Competitive Feature-Free Approach for Automated Algorithm Selection on the Traveling Salesperson Problem}}}, doi = {{10.1007/978-3-030-58112-1_4}}, year = {{2020}}, } @article{48848, abstract = {{We build upon a recently proposed multi-objective view onto performance measurement of single-objective stochastic solvers. The trade-off between the fraction of failed runs and the mean runtime of successful runs \textendash both to be minimized \textendash is directly analyzed based on a study on algorithm selection of inexact state-of-the-art solvers for the famous Traveling Salesperson Problem (TSP). Moreover, we adopt the hypervolume indicator (HV) commonly used in multi-objective optimization for simultaneously assessing both conflicting objectives and investigate relations to commonly used performance indicators, both theoretically and empirically. Next to Penalized Average Runtime (PAR) and Penalized Quantile Runtime (PQR), the HV measure is used as a core concept within the construction of per-instance algorithm selection models offering interesting insights into complementary behavior of inexact TSP solvers. \textbullet The multi-objective perspective is naturally generalizable to multiple objectives. \textbullet Proof of relationship between HV and the PAR in the considered bi-objective space. \textbullet New insights into complementary behavior of stochastic optimization algorithms.}}, author = {{Bossek, Jakob and Kerschke, Pascal and Trautmann, Heike}}, issn = {{1568-4946}}, journal = {{Applied Soft Computing}}, keywords = {{Algorithm selection, Combinatorial optimization, Multi-objective optimization, Performance measurement, Traveling Salesperson Problem}}, number = {{C}}, title = {{{A Multi-Objective Perspective on Performance Assessment and Automated Selection of Single-Objective Optimization Algorithms}}}, doi = {{10.1016/j.asoc.2019.105901}}, volume = {{88}}, year = {{2020}}, } @article{48836, author = {{Bartz-Beielstein, Thomas and Doerr, Carola and van den Berg, Daan and Bossek, Jakob and Chandrasekaran, Sowmya and Eftimov, Tome and Fischbach, Andreas and Kerschke, Pascal and Cava, William La and Lopez-Ibanez, Manuel and Malan, Katherine M. and Moore, Jason H. and Naujoks, Boris and Orzechowski, Patryk and Volz, Vanessa and Wagner, Markus and Weise, Thomas}}, journal = {{Corr}}, title = {{{Benchmarking in Optimization: Best Practice and Open Issues}}}, year = {{2020}}, } @inproceedings{46330, abstract = {{In this work we focus on the well-known Euclidean Traveling Salesperson Problem (TSP) and two highly competitive inexact heuristic TSP solvers, EAX and LKH, in the context of per-instance algorithm selection (AS). We evolve instances with 1000 nodes where the solvers show strongly different performance profiles. These instances serve as a basis for an exploratory study on the identification of well-discriminating problem characteristics (features). Our results in a nutshell: we show that even though (1) promising features exist, (2) these are in line with previous results from the literature, and (3) models trained with these features are more accurate than models adopting sophisticated feature selection methods, the advantage is not close to the virtual best solver in terms of penalized average runtime and so is the performance gain over the single best solver. However, we show that a feature-free deep neural network based approach solely based on visual representation of the instances already matches classical AS model results and thus shows huge potential for future studies.}}, author = {{Seiler, Moritz and Pohl, Janina and Bossek, Jakob and Kerschke, Pascal and Trautmann, Heike}}, booktitle = {{Proceedings of the 16$^th$ International Conference on Parallel Problem Solving from Nature (PPSN XVI)}}, editor = {{Bäck, Thomas and Preuss, Mike and Deutz, André and Wang, Hao and Doerr, Carola and Emmerich, Michael and Trautmann, Heike}}, pages = {{48–64}}, title = {{{Deep Learning as a Competitive Feature-Free Approach for Automated Algorithm Selection on the Traveling Salesperson Problem}}}, doi = {{10.1007/978-3-030-58112-1_4}}, year = {{2020}}, } @article{46334, abstract = {{We build upon a recently proposed multi-objective view onto performance measurement of single-objective stochastic solvers. The trade-off between the fraction of failed runs and the mean runtime of successful runs – both to be minimized – is directly analyzed based on a study on algorithm selection of inexact state-of-the-art solvers for the famous Traveling Salesperson Problem (TSP). Moreover, we adopt the hypervolume indicator (HV) commonly used in multi-objective optimization for simultaneously assessing both conflicting objectives and investigate relations to commonly used performance indicators, both theoretically and empirically. Next to Penalized Average Runtime (PAR) and Penalized Quantile Runtime (PQR), the HV measure is used as a core concept within the construction of per-instance algorithm selection models offering interesting insights into complementary behavior of inexact TSP solvers.}}, author = {{Bossek, Jakob and Kerschke, Pascal and Trautmann, Heike}}, issn = {{1568-4946}}, journal = {{Applied Soft Computing}}, keywords = {{Algorithm selection, Multi-objective optimization, Performance measurement, Combinatorial optimization, Traveling Salesperson Problem}}, pages = {{105901}}, title = {{{A multi-objective perspective on performance assessment and automated selection of single-objective optimization algorithms}}}, doi = {{https://doi.org/10.1016/j.asoc.2019.105901}}, volume = {{88}}, year = {{2020}}, } @inproceedings{46322, abstract = {{We consider a dynamic bi-objective vehicle routing problem, where a subset of customers ask for service over time. Therein, the distance traveled by a single vehicle and the number of unserved dynamic requests is minimized by a dynamic evolutionary multi-objective algorithm (DEMOA), which operates on discrete time windows (eras). A decision is made at each era by a decision-maker, thus any decision depends on irreversible decisions made in foregoing eras. To understand effects of sequences of decision-making and interactions/dependencies between decisions made, we conduct a series of experiments. More precisely, we fix a set of decision-maker preferences D and the number of eras n t and analyze all |D| nt combinations of decision-maker options. We find that for random uniform instances (a) the final selected solutions mainly depend on the final decision and not on the decision history, (b) solutions are quite robust with respect to the number of unvisited dynamic customers, and (c) solutions of the dynamic approach can even dominate solutions obtained by a clairvoyant EMOA. In contrast, for instances with clustered customers, we observe a strong dependency on decision-making history as well as more variance in solution diversity.}}, author = {{Bossek, Jakob and Grimme, Christian and Rudolph, Günter and Trautmann, Heike}}, booktitle = {{Proceedings of the IEEE Congress on Evolutionary Computation (CEC)}}, pages = {{1–8}}, title = {{{Towards Decision Support in Dynamic Bi-Objective Vehicle Routing}}}, doi = {{10.1109/CEC48606.2020.9185778}}, year = {{2020}}, } @inproceedings{46324, abstract = {{The Traveling-Salesperson-Problem (TSP) is arguably one of the best-known NP-hard combinatorial optimization problems. The two sophisticated heuristic solvers LKH and EAX and respective (restart) variants manage to calculate close-to optimal or even optimal solutions, also for large instances with several thousand nodes in reasonable time. In this work we extend existing benchmarking studies by addressing anytime behaviour of inexact TSP solvers based on empirical runtime distributions leading to an increased understanding of solver behaviour and the respective relation to problem hardness. It turns out that performance ranking of solvers is highly dependent on the focused approximation quality. Insights on intersection points of performances offer huge potential for the construction of hybridized solvers depending on instance features. Moreover, instance features tailored to anytime performance and corresponding performance indicators will highly improve automated algorithm selection models by including comprehensive information on solver quality.}}, author = {{Bossek, Jakob and Kerschke, Pascal and Trautmann, Heike}}, booktitle = {{Proceedings of the IEEE Congress on Evolutionary Computation (CEC)}}, pages = {{1–8}}, publisher = {{IEEE}}, title = {{{Anytime Behavior of Inexact TSP Solvers and Perspectives for Automated Algorithm Selection}}}, year = {{2020}}, } @inproceedings{46323, abstract = {{In practice, e.g. in delivery and service scenarios, Vehicle-Routing-Problems (VRPs) often imply repeated decision making on dynamic customer requests. As in classical VRPs, tours have to be planned short while the number of serviced customers has to be maximized at the same time resulting in a multi-objective problem. Beyond that, however, dynamic requests lead to the need for re-planning of not yet realized tour parts, while already realized tour parts are irreversible. In this paper we study this type of bi-objective dynamic VRP including sequential decision making and concurrent realization of decisions. We adopt a recently proposed Dynamic Evolutionary Multi-Objective Algorithm (DEMOA) for a related VRP problem and extend it to the more realistic (here considered) scenario of multiple vehicles. We empirically show that our DEMOA is competitive with a multi-vehicle offline and clairvoyant variant of the proposed DEMOA as well as with the dynamic single-vehicle approach proposed earlier.}}, author = {{Bossek, Jakob and Grimme, Christian and Trautmann, Heike}}, booktitle = {{Proceedings of the Genetic and Evolutionary Computation Conference (GECCO ’20)}}, pages = {{166–174}}, publisher = {{ACM}}, title = {{{Dynamic Bi-Objective Routing of Multiple Vehicles}}}, year = {{2020}}, } @inproceedings{48841, abstract = {{We tackle a bi-objective dynamic orienteering problem where customer requests arise as time passes by. The goal is to minimize the tour length traveled by a single delivery vehicle while simultaneously keeping the number of dismissed dynamic customers to a minimum. We propose a dynamic Evolutionary Multi-Objective Algorithm which is grounded on insights gained from a previous series of work on an a-posteriori version of the problem, where all request times are known in advance. In our experiments, we simulate different decision maker strategies and evaluate the development of the Pareto-front approximations on exemplary problem instances. It turns out, that despite severely reduced computational budget and no oracle-knowledge of request times the dynamic EMOA is capable of producing approximations which partially dominate the results of the a-posteriori EMOA and dynamic integer linear programming strategies.}}, author = {{Bossek, Jakob and Grimme, Christian and Meisel, Stephan and Rudolph, Günter and Trautmann, Heike}}, booktitle = {{Evolutionary Multi-Criterion Optimization (EMO)}}, editor = {{Deb, Kalyanmoy and Goodman, Erik and Coello Coello, Carlos A. and Klamroth, Kathrin and Miettinen, Kaisa and Mostaghim, Sanaz and Reed, Patrick}}, isbn = {{978-3-030-12598-1}}, keywords = {{Combinatorial optimization, Dynamic optimization, Metaheuristics, Multi-objective optimization, Vehicle routing}}, pages = {{516–528}}, publisher = {{Springer International Publishing}}, title = {{{Bi-Objective Orienteering: Towards a Dynamic Multi-objective Evolutionary Algorithm}}}, doi = {{10.1007/978-3-030-12598-1_41}}, year = {{2019}}, } @inproceedings{48842, abstract = {{Evolutionary algorithms have successfully been applied to evolve problem instances that exhibit a significant difference in performance for a given algorithm or a pair of algorithms inter alia for the Traveling Salesperson Problem (TSP). Creating a large variety of instances is crucial for successful applications in the blooming field of algorithm selection. In this paper, we introduce new and creative mutation operators for evolving instances of the TSP. We show that adopting those operators in an evolutionary algorithm allows for the generation of benchmark sets with highly desirable properties: (1) novelty by clear visual distinction to established benchmark sets in the field, (2) visual and quantitative diversity in the space of TSP problem characteristics, and (3) significant performance differences with respect to the restart versions of heuristic state-of-the-art TSP solvers EAX and LKH. The important aspect of diversity is addressed and achieved solely by the proposed mutation operators and not enforced by explicit diversity preservation.}}, author = {{Bossek, Jakob and Kerschke, Pascal and Neumann, Aneta and Wagner, Markus and Neumann, Frank and Trautmann, Heike}}, booktitle = {{Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms}}, isbn = {{978-1-4503-6254-2}}, keywords = {{benchmarking, instance features, optimization, problem generation, traveling salesperson problem}}, pages = {{58–71}}, publisher = {{Association for Computing Machinery}}, title = {{{Evolving Diverse TSP Instances by Means of Novel and Creative Mutation Operators}}}, doi = {{10.1145/3299904.3340307}}, year = {{2019}}, } @inproceedings{48843, abstract = {{We contribute to the theoretical understanding of randomized search heuristics for dynamic problems. We consider the classical graph coloring problem and investigate the dynamic setting where edges are added to the current graph. We then analyze the expected time for randomized search heuristics to recompute high quality solutions. This includes the (1+1) EA and RLS in a setting where the number of colors is bounded and we are minimizing the number of conflicts as well as iterated local search algorithms that use an unbounded color palette and aim to use the smallest colors and - as a consequence - the smallest number of colors. We identify classes of bipartite graphs where reoptimization is as hard as or even harder than optimization from scratch, i. e. starting with a random initialization. Even adding a single edge can lead to hard symmetry problems. However, graph classes that are hard for one algorithm turn out to be easy for others. In most cases our bounds show that reoptimization is faster than optimizing from scratch. Furthermore, we show how to speed up computations by using problem specific operators concentrating on parts of the graph where changes have occurred.}}, author = {{Bossek, Jakob and Neumann, Frank and Peng, Pan and Sudholt, Dirk}}, booktitle = {{Proceedings of the Genetic and Evolutionary Computation Conference}}, isbn = {{978-1-4503-6111-8}}, keywords = {{dynamic optimization, evolutionary algorithms, running time analysis, theory}}, pages = {{1443–1451}}, publisher = {{Association for Computing Machinery}}, title = {{{Runtime Analysis of Randomized Search Heuristics for Dynamic Graph Coloring}}}, doi = {{10.1145/3321707.3321792}}, year = {{2019}}, } @inproceedings{48840, abstract = {{Research has shown that for many single-objective graph problems where optimum solutions are composed of low weight sub-graphs, such as the minimum spanning tree problem (MST), mutation operators favoring low weight edges show superior performance. Intuitively, similar observations should hold for multi-criteria variants of such problems. In this work, we focus on the multi-criteria MST problem. A thorough experimental study is conducted where we estimate the probability of edges being part of non-dominated spanning trees as a function of the edges’ non-domination level or domination count, respectively. Building on gained insights, we propose several biased one-edge-exchange mutation operators that differ in the used edge-selection probability distribution (biased towards edges of low rank). Our empirical analysis shows that among different graph types (dense and sparse) and edge weight types (both uniformly random and combinations of Euclidean and uniformly random) biased edge-selection strategies perform superior in contrast to the baseline uniform edge-selection. Our findings are in particular strong for dense graphs.}}, author = {{Bossek, Jakob and Grimme, Christian and Neumann, Frank}}, booktitle = {{Proceedings of the Genetic and Evolutionary Computation Conference}}, isbn = {{978-1-4503-6111-8}}, keywords = {{biased mutation, combinatorial optimization, minimum spanning tree, multi-objective optimization}}, pages = {{516–523}}, publisher = {{Association for Computing Machinery}}, title = {{{On the Benefits of Biased Edge-Exchange Mutation for the Multi-Criteria Spanning Tree Problem}}}, doi = {{10.1145/3321707.3321818}}, year = {{2019}}, } @inproceedings{48858, abstract = {{The $$\textbackslash mathcal NP$$-hard multi-criteria shortest path problem (mcSPP) is of utmost practical relevance, e.~g., in navigation system design and logistics. We address the problem of approximating the Pareto-front of the mcSPP with sum objectives. We do so by proposing a new mutation operator for multi-objective evolutionary algorithms that solves single-objective versions of the shortest path problem on subgraphs. A rigorous empirical benchmark on a diverse set of problem instances shows the effectiveness of the approach in comparison to a well-known mutation operator in terms of convergence speed and approximation quality. In addition, we glance at the neighbourhood structure and similarity of obtained Pareto-optimal solutions and derive promising directions for future work.}}, author = {{Bossek, Jakob and Grimme, Christian}}, booktitle = {{Learning and Intelligent Optimization}}, editor = {{Battiti, Roberto and Brunato, Mauro and Kotsireas, Ilias and Pardalos, Panos M.}}, isbn = {{978-3-030-05348-2}}, pages = {{184–198}}, publisher = {{Springer International Publishing}}, title = {{{Solving Scalarized Subproblems within Evolutionary Algorithms for Multi-criteria Shortest Path Problems}}}, doi = {{10.1007/978-3-030-05348-2_17}}, year = {{2019}}, } @inproceedings{48870, abstract = {{The edge coloring problem asks for an assignment of colors to edges of a graph such that no two incident edges share the same color and the number of colors is minimized. It is known that all graphs with maximum degree {$\Delta$} can be colored with {$\Delta$} or {$\Delta$} + 1 colors, but it is NP-hard to determine whether {$\Delta$} colors are sufficient. We present the first runtime analysis of evolutionary algorithms (EAs) for the edge coloring problem. Simple EAs such as RLS and (1+1) EA efficiently find (2{$\Delta$} - 1)-colorings on arbitrary graphs and optimal colorings for even and odd cycles, paths, star graphs and arbitrary trees. A partial analysis for toroids also suggests efficient runtimes in bipartite graphs with many cycles. Experiments support these findings and investigate additional graph classes such as hypercubes, complete graphs and complete bipartite graphs. Theoretical and experimental results suggest that simple EAs find optimal colorings for all these graph classes in expected time O({$\Delta\mathscrl$}2m log m), where m is the number of edges and {$\mathscrl$} is the length of the longest simple path in the graph.}}, author = {{Bossek, Jakob and Sudholt, Dirk}}, booktitle = {{Proceedings of the 15th ACM/SIGEVO Conference on Foundations of Genetic Algorithms}}, isbn = {{978-1-4503-6254-2}}, keywords = {{edge coloring problem, runtime analysis}}, pages = {{102–115}}, publisher = {{Association for Computing Machinery}}, title = {{{Time Complexity Analysis of RLS and (1 + 1) EA for the Edge Coloring Problem}}}, doi = {{10.1145/3299904.3340311}}, year = {{2019}}, } @inproceedings{48875, abstract = {{A multiobjective perspective onto common performance measures such as the PAR10 score or the expected runtime of single-objective stochastic solvers is presented by directly investigating the tradeoff between the fraction of failed runs and the average runtime. Multi-objective indicators operating in the bi-objective space allow for an overall performance comparison on a set of instances paving the way for instance-based automated algorithm selection techniques.}}, author = {{Bossek, Jakob and Trautmann, Heike}}, booktitle = {{Learning and Intelligent Optimization}}, editor = {{Battiti, Roberto and Brunato, Mauro and Kotsireas, Ilias and Pardalos, Panos M.}}, isbn = {{978-3-030-05348-2}}, keywords = {{Algorithm selection, Performance measurement}}, pages = {{215–219}}, publisher = {{Springer International Publishing}}, title = {{{Multi-Objective Performance Measurement: Alternatives to PAR10 and Expected Running Time}}}, doi = {{10.1007/978-3-030-05348-2_19}}, year = {{2019}}, } @article{48877, abstract = {{OpenML is an online machine learning platform where researchers can easily share data, machine learning tasks and experiments as well as organize them online to work and collaborate more efficiently. In this paper, we present an R package to interface with the OpenML platform and illustrate its usage in combination with the machine learning R package mlr (Bischl et al. J Mach Learn Res 17(170):1—5, 2016). We show how the OpenML package allows R users to easily search, download and upload data sets and machine learning tasks. Furthermore, we also show how to upload results of experiments, share them with others and download results from other users. Beyond ensuring reproducibility of results, the OpenML platform automates much of the drudge work, speeds up research, facilitates collaboration and increases the users’ visibility online.}}, author = {{Casalicchio, Giuseppe and Bossek, Jakob and Lang, Michel and Kirchhoff, Dominik and Kerschke, Pascal and Hofner, Benjamin and Seibold, Heidi and Vanschoren, Joaquin and Bischl, Bernd}}, issn = {{0943-4062}}, journal = {{Computational Statistics}}, keywords = {{Databases, Machine learning, R, Reproducible research}}, number = {{3}}, pages = {{977–991}}, title = {{{OpenML: An R Package to Connect to the Machine Learning Platform OpenML}}}, doi = {{10.1007/s00180-017-0742-2}}, volume = {{34}}, year = {{2019}}, }