@inbook{45884, author = {{Hanselle, Jonas Manuel and Hüllermeier, Eyke and Mohr, Felix and Ngonga Ngomo, Axel-Cyrille and Sherif, Mohamed and Tornede, Alexander and Wever, Marcel Dominik}}, booktitle = {{On-The-Fly Computing -- Individualized IT-services in dynamic markets}}, editor = {{Haake, Claus-Jochen and Meyer auf der Heide, Friedhelm and Platzner, Marco and Wachsmuth, Henning and Wehrheim, Heike}}, pages = {{85--104}}, publisher = {{Heinz Nixdorf Institut, Universität Paderborn}}, title = {{{Configuration and Evaluation}}}, doi = {{10.5281/zenodo.8068466}}, volume = {{412}}, year = {{2023}}, } @unpublished{30868, abstract = {{Algorithm configuration (AC) is concerned with the automated search of the most suitable parameter configuration of a parametrized algorithm. There is currently a wide variety of AC problem variants and methods proposed in the literature. Existing reviews do not take into account all derivatives of the AC problem, nor do they offer a complete classification scheme. To this end, we introduce taxonomies to describe the AC problem and features of configuration methods, respectively. We review existing AC literature within the lens of our taxonomies, outline relevant design choices of configuration approaches, contrast methods and problem variants against each other, and describe the state of AC in industry. Finally, our review provides researchers and practitioners with a look at future research directions in the field of AC.}}, author = {{Schede, Elias and Brandt, Jasmin and Tornede, Alexander and Wever, Marcel Dominik and Bengs, Viktor and Hüllermeier, Eyke and Tierney, Kevin}}, booktitle = {{arXiv:2202.01651}}, title = {{{A Survey of Methods for Automated Algorithm Configuration}}}, year = {{2022}}, } @unpublished{30865, abstract = {{The problem of selecting an algorithm that appears most suitable for a specific instance of an algorithmic problem class, such as the Boolean satisfiability problem, is called instance-specific algorithm selection. Over the past decade, the problem has received considerable attention, resulting in a number of different methods for algorithm selection. Although most of these methods are based on machine learning, surprisingly little work has been done on meta learning, that is, on taking advantage of the complementarity of existing algorithm selection methods in order to combine them into a single superior algorithm selector. In this paper, we introduce the problem of meta algorithm selection, which essentially asks for the best way to combine a given set of algorithm selectors. We present a general methodological framework for meta algorithm selection as well as several concrete learning methods as instantiations of this framework, essentially combining ideas of meta learning and ensemble learning. In an extensive experimental evaluation, we demonstrate that ensembles of algorithm selectors can significantly outperform single algorithm selectors and have the potential to form the new state of the art in algorithm selection.}}, author = {{Tornede, Alexander and Gehring, Lukas and Tornede, Tanja and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Machine Learning}}, title = {{{Algorithm Selection on a Meta Level}}}, year = {{2022}}, } @article{33090, abstract = {{AbstractHeated tool butt welding is a method often used for joining thermoplastics, especially when the components are made out of different materials. The quality of the connection between the components crucially depends on a suitable choice of the parameters of the welding process, such as heating time, temperature, and the precise way how the parts are then welded. Moreover, when different materials are to be joined, the parameter values need to be tailored to the specifics of the respective material. To this end, in this paper, three approaches to tailor the parameter values to optimize the quality of the connection are compared: a heuristic by Potente, statistical experimental design, and Bayesian optimization. With the suitability for practice in mind, a series of experiments are carried out with these approaches, and their capabilities of proposing well-performing parameter values are investigated. As a result, Bayesian optimization is found to yield peak performance, but the costs for optimization are substantial. In contrast, the Potente heuristic does not require any experimentation and recommends parameter values with competitive quality.}}, author = {{Gevers, Karina and Tornede, Alexander and Wever, Marcel Dominik and Schöppner, Volker and Hüllermeier, Eyke}}, issn = {{0043-2288}}, journal = {{Welding in the World}}, keywords = {{Metals and Alloys, Mechanical Engineering, Mechanics of Materials}}, publisher = {{Springer Science and Business Media LLC}}, title = {{{A comparison of heuristic, statistical, and machine learning methods for heated tool butt welding of two different materials}}}, doi = {{10.1007/s40194-022-01339-9}}, year = {{2022}}, } @article{21004, abstract = {{Automated machine learning (AutoML) supports the algorithmic construction and data-specific customization of machine learning pipelines, including the selection, combination, and parametrization of machine learning algorithms as main constituents. Generally speaking, AutoML approaches comprise two major components: a search space model and an optimizer for traversing the space. Recent approaches have shown impressive results in the realm of supervised learning, most notably (single-label) classification (SLC). Moreover, first attempts at extending these approaches towards multi-label classification (MLC) have been made. While the space of candidate pipelines is already huge in SLC, the complexity of the search space is raised to an even higher power in MLC. One may wonder, therefore, whether and to what extent optimizers established for SLC can scale to this increased complexity, and how they compare to each other. This paper makes the following contributions: First, we survey existing approaches to AutoML for MLC. Second, we augment these approaches with optimizers not previously tried for MLC. Third, we propose a benchmarking framework that supports a fair and systematic comparison. Fourth, we conduct an extensive experimental study, evaluating the methods on a suite of MLC problems. We find a grammar-based best-first search to compare favorably to other optimizers.}}, author = {{Wever, Marcel Dominik and Tornede, Alexander and Mohr, Felix and Hüllermeier, Eyke}}, issn = {{0162-8828}}, journal = {{IEEE Transactions on Pattern Analysis and Machine Intelligence}}, keywords = {{Automated Machine Learning, Multi Label Classification, Hierarchical Planning, Bayesian Optimization}}, pages = {{1--1}}, title = {{{AutoML for Multi-Label Classification: Overview and Empirical Evaluation}}}, doi = {{10.1109/tpami.2021.3051276}}, year = {{2021}}, } @article{21092, abstract = {{Automated Machine Learning (AutoML) seeks to automatically find so-called machine learning pipelines that maximize the prediction performance when being used to train a model on a given dataset. One of the main and yet open challenges in AutoML is an effective use of computational resources: An AutoML process involves the evaluation of many candidate pipelines, which are costly but often ineffective because they are canceled due to a timeout. In this paper, we present an approach to predict the runtime of two-step machine learning pipelines with up to one pre-processor, which can be used to anticipate whether or not a pipeline will time out. Separate runtime models are trained offline for each algorithm that may be used in a pipeline, and an overall prediction is derived from these models. We empirically show that the approach increases successful evaluations made by an AutoML tool while preserving or even improving on the previously best solutions.}}, author = {{Mohr, Felix and Wever, Marcel Dominik and Tornede, Alexander and Hüllermeier, Eyke}}, journal = {{IEEE Transactions on Pattern Analysis and Machine Intelligence}}, publisher = {{IEEE}}, title = {{{Predicting Machine Learning Pipeline Runtimes in the Context of Automated Machine Learning}}}, year = {{2021}}, } @inproceedings{21570, author = {{Tornede, Tanja and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the Genetic and Evolutionary Computation Conference}}, title = {{{Coevolution of Remaining Useful Lifetime Estimation Pipelines for Automated Predictive Maintenance}}}, year = {{2021}}, } @inproceedings{22913, author = {{Hüllermeier, Eyke and Mohr, Felix and Tornede, Alexander and Wever, Marcel Dominik}}, location = {{Bilbao (Virtual)}}, title = {{{Automated Machine Learning, Bounded Rationality, and Rational Metareasoning}}}, year = {{2021}}, } @inproceedings{22914, author = {{Mohr, Felix and Wever, Marcel Dominik}}, location = {{Virtual}}, title = {{{Replacing the Ex-Def Baseline in AutoML by Naive AutoML}}}, year = {{2021}}, } @unpublished{30866, abstract = {{Automated machine learning (AutoML) strives for the automatic configuration of machine learning algorithms and their composition into an overall (software) solution - a machine learning pipeline - tailored to the learning task (dataset) at hand. Over the last decade, AutoML has developed into an independent research field with hundreds of contributions. While AutoML offers many prospects, it is also known to be quite resource-intensive, which is one of its major points of criticism. The primary cause for a high resource consumption is that many approaches rely on the (costly) evaluation of many machine learning pipelines while searching for good candidates. This problem is amplified in the context of research on AutoML methods, due to large scale experiments conducted with many datasets and approaches, each of them being run with several repetitions to rule out random effects. In the spirit of recent work on Green AI, this paper is written in an attempt to raise the awareness of AutoML researchers for the problem and to elaborate on possible remedies. To this end, we identify four categories of actions the community may take towards more sustainable research on AutoML, i.e. Green AutoML: design of AutoML systems, benchmarking, transparency and research incentives.}}, author = {{Tornede, Tanja and Tornede, Alexander and Hanselle, Jonas Manuel and Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{arXiv:2111.05850}}, title = {{{Towards Green Automated Machine Learning: Status Quo and Future Directions}}}, year = {{2021}}, } @phdthesis{27284, author = {{Wever, Marcel Dominik}}, title = {{{Automated Machine Learning for Multi-Label Classification}}}, doi = {{10.17619/UNIPB/1-1302}}, year = {{2021}}, } @inproceedings{21198, author = {{Hanselle, Jonas Manuel and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, location = {{Delhi, India}}, title = {{{Algorithm Selection as Superset Learning: Constructing Algorithm Selectors from Imprecise Performance Data}}}, year = {{2021}}, } @inproceedings{17407, author = {{Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Discovery Science}}, title = {{{Extreme Algorithm Selection with Dyadic Feature Representation}}}, year = {{2020}}, } @inproceedings{17408, author = {{Hanselle, Jonas Manuel and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{KI 2020: Advances in Artificial Intelligence}}, title = {{{Hybrid Ranking and Regression for Algorithm Selection}}}, year = {{2020}}, } @inproceedings{17424, author = {{Tornede, Tanja and Tornede, Alexander and Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the ECMLPKDD 2020}}, title = {{{AutoML for Predictive Maintenance: One Tool to RUL Them All}}}, doi = {{10.1007/978-3-030-66770-2_8}}, year = {{2020}}, } @unpublished{17605, abstract = {{Syntactic annotation of corpora in the form of part-of-speech (POS) tags is a key requirement for both linguistic research and subsequent automated natural language processing (NLP) tasks. This problem is commonly tackled using machine learning methods, i.e., by training a POS tagger on a sufficiently large corpus of labeled data. While the problem of POS tagging can essentially be considered as solved for modern languages, historical corpora turn out to be much more difficult, especially due to the lack of native speakers and sparsity of training data. Moreover, most texts have no sentences as we know them today, nor a common orthography. These irregularities render the task of automated POS tagging more difficult and error-prone. Under these circumstances, instead of forcing the POS tagger to predict and commit to a single tag, it should be enabled to express its uncertainty. In this paper, we consider POS tagging within the framework of set-valued prediction, which allows the POS tagger to express its uncertainty via predicting a set of candidate POS tags instead of guessing a single one. The goal is to guarantee a high confidence that the correct POS tag is included while keeping the number of candidates small. In our experimental study, we find that extending state-of-the-art POS taggers to set-valued prediction yields more precise and robust taggings, especially for unknown words, i.e., words not occurring in the training data.}}, author = {{Heid, Stefan Helmut and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Journal of Data Mining and Digital Humanities}}, publisher = {{episciences}}, title = {{{Reliable Part-of-Speech Tagging of Historical Corpora through Set-Valued Prediction}}}, year = {{2020}}, } @inproceedings{20306, author = {{Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Workshop MetaLearn 2020 @ NeurIPS 2020}}, location = {{Online}}, title = {{{Towards Meta-Algorithm Selection}}}, year = {{2020}}, } @inproceedings{18276, abstract = {{Algorithm selection (AS) deals with the automatic selection of an algorithm from a fixed set of candidate algorithms most suitable for a specific instance of an algorithmic problem class, where "suitability" often refers to an algorithm's runtime. Due to possibly extremely long runtimes of candidate algorithms, training data for algorithm selection models is usually generated under time constraints in the sense that not all algorithms are run to completion on all instances. Thus, training data usually comprises censored information, as the true runtime of algorithms timed out remains unknown. However, many standard AS approaches are not able to handle such information in a proper way. On the other side, survival analysis (SA) naturally supports censored data and offers appropriate ways to use such data for learning distributional models of algorithm runtime, as we demonstrate in this work. We leverage such models as a basis of a sophisticated decision-theoretic approach to algorithm selection, which we dub Run2Survive. Moreover, taking advantage of a framework of this kind, we advocate a risk-averse approach to algorithm selection, in which the avoidance of a timeout is given high priority. In an extensive experimental study with the standard benchmark ASlib, our approach is shown to be highly competitive and in many cases even superior to state-of-the-art AS approaches.}}, author = {{Tornede, Alexander and Wever, Marcel Dominik and Werner, Stefan and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{ACML 2020}}, location = {{Bangkok, Thailand}}, title = {{{Run2Survive: A Decision-theoretic Approach to Algorithm Selection based on Survival Analysis}}}, year = {{2020}}, } @inproceedings{15629, abstract = {{In multi-label classification (MLC), each instance is associated with a set of class labels, in contrast to standard classification where an instance is assigned a single label. Binary relevance (BR) learning, which reduces a multi-label to a set of binary classification problems, one per label, is arguably the most straight-forward approach to MLC. In spite of its simplicity, BR proved to be competitive to more sophisticated MLC methods, and still achieves state-of-the-art performance for many loss functions. Somewhat surprisingly, the optimal choice of the base learner for tackling the binary classification problems has received very little attention so far. Taking advantage of the label independence assumption inherent to BR, we propose a label-wise base learner selection method optimizing label-wise macro averaged performance measures. In an extensive experimental evaluation, we find that or approach, called LiBRe, can significantly improve generalization performance.}}, author = {{Wever, Marcel Dominik and Tornede, Alexander and Mohr, Felix and Hüllermeier, Eyke}}, location = {{Konstanz, Germany}}, publisher = {{Springer}}, title = {{{LiBRe: Label-Wise Selection of Base Learners in Binary Relevance for Multi-Label Classification}}}, year = {{2020}}, } @article{15025, abstract = {{In software engineering, the imprecise requirements of a user are transformed to a formal requirements specification during the requirements elicitation process. This process is usually guided by requirements engineers interviewing the user. We want to partially automate this first step of the software engineering process in order to enable users to specify a desired software system on their own. With our approach, users are only asked to provide exemplary behavioral descriptions. The problem of synthesizing a requirements specification from examples can partially be reduced to the problem of grammatical inference, to which we apply an active coevolutionary learning approach. However, this approach would usually require many feedback queries to be sent to the user. In this work, we extend and generalize our active learning approach to receive knowledge from multiple oracles, also known as proactive learning. The ‘user oracle’ represents input received from the user and the ‘knowledge oracle’ represents available, formalized domain knowledge. We call our two-oracle approach the ‘first apply knowledge then query’ (FAKT/Q) algorithm. We compare FAKT/Q to the active learning approach and provide an extensive benchmark evaluation. As result we find that the number of required user queries is reduced and the inference process is sped up significantly. Finally, with so-called On-The-Fly Markets, we present a motivation and an application of our approach where such knowledge is available.}}, author = {{Wever, Marcel Dominik and van Rooijen, Lorijn and Hamann, Heiko}}, journal = {{Evolutionary Computation}}, number = {{2}}, pages = {{165–193}}, publisher = {{MIT Press Journals}}, title = {{{Multi-Oracle Coevolutionary Learning of Requirements Specifications from Examples in On-The-Fly Markets}}}, doi = {{10.1162/evco_a_00266}}, volume = {{28}}, year = {{2020}}, }