@phdthesis{45780, author = {{Tornede, Alexander}}, title = {{{Advanced Algorithm Selection with Machine Learning: Handling Large Algorithm Sets, Learning From Censored Data, and Simplyfing Meta Level Decisions}}}, doi = {{10.17619/UNIPB/1-1780 }}, year = {{2023}}, } @article{21600, abstract = {{Many problems in science and engineering require an efficient numerical approximation of integrals or solutions to differential equations. For systems with rapidly changing dynamics, an equidistant discretization is often inadvisable as it results in prohibitively large errors or computational effort. To this end, adaptive schemes, such as solvers based on Runge–Kutta pairs, have been developed which adapt the step size based on local error estimations at each step. While the classical schemes apply very generally and are highly efficient on regular systems, they can behave suboptimally when an inefficient step rejection mechanism is triggered by structurally complex systems such as chaotic systems. To overcome these issues, we propose a method to tailor numerical schemes to the problem class at hand. This is achieved by combining simple, classical quadrature rules or ODE solvers with data-driven time-stepping controllers. Compared with learning solution operators to ODEs directly, it generalizes better to unseen initial data as our approach employs classical numerical schemes as base methods. At the same time it can make use of identified structures of a problem class and, therefore, outperforms state-of-the-art adaptive schemes. Several examples demonstrate superior efficiency. Source code is available at https://github.com/lueckem/quadrature-ML.}}, author = {{Dellnitz, Michael and Hüllermeier, Eyke and Lücke, Marvin and Ober-Blöbaum, Sina and Offen, Christian and Peitz, Sebastian and Pfannschmidt, Karlson}}, journal = {{SIAM Journal on Scientific Computing}}, number = {{2}}, pages = {{A579--A595}}, title = {{{Efficient time stepping for numerical integration using reinforcement learning}}}, doi = {{10.1137/21M1412682}}, volume = {{45}}, year = {{2023}}, } @inproceedings{24382, author = {{Gevers, Karina and Schöppner, Volker and Hüllermeier, Eyke}}, location = {{online}}, title = {{{Heated tool butt welding of two different materials – Established methods versus artificial intelligence}}}, year = {{2021}}, } @article{21004, abstract = {{Automated machine learning (AutoML) supports the algorithmic construction and data-specific customization of machine learning pipelines, including the selection, combination, and parametrization of machine learning algorithms as main constituents. Generally speaking, AutoML approaches comprise two major components: a search space model and an optimizer for traversing the space. Recent approaches have shown impressive results in the realm of supervised learning, most notably (single-label) classification (SLC). Moreover, first attempts at extending these approaches towards multi-label classification (MLC) have been made. While the space of candidate pipelines is already huge in SLC, the complexity of the search space is raised to an even higher power in MLC. One may wonder, therefore, whether and to what extent optimizers established for SLC can scale to this increased complexity, and how they compare to each other. This paper makes the following contributions: First, we survey existing approaches to AutoML for MLC. Second, we augment these approaches with optimizers not previously tried for MLC. Third, we propose a benchmarking framework that supports a fair and systematic comparison. Fourth, we conduct an extensive experimental study, evaluating the methods on a suite of MLC problems. We find a grammar-based best-first search to compare favorably to other optimizers.}}, author = {{Wever, Marcel Dominik and Tornede, Alexander and Mohr, Felix and Hüllermeier, Eyke}}, issn = {{0162-8828}}, journal = {{IEEE Transactions on Pattern Analysis and Machine Intelligence}}, keywords = {{Automated Machine Learning, Multi Label Classification, Hierarchical Planning, Bayesian Optimization}}, pages = {{1--1}}, title = {{{AutoML for Multi-Label Classification: Overview and Empirical Evaluation}}}, doi = {{10.1109/tpami.2021.3051276}}, year = {{2021}}, } @article{21092, abstract = {{Automated Machine Learning (AutoML) seeks to automatically find so-called machine learning pipelines that maximize the prediction performance when being used to train a model on a given dataset. One of the main and yet open challenges in AutoML is an effective use of computational resources: An AutoML process involves the evaluation of many candidate pipelines, which are costly but often ineffective because they are canceled due to a timeout. In this paper, we present an approach to predict the runtime of two-step machine learning pipelines with up to one pre-processor, which can be used to anticipate whether or not a pipeline will time out. Separate runtime models are trained offline for each algorithm that may be used in a pipeline, and an overall prediction is derived from these models. We empirically show that the approach increases successful evaluations made by an AutoML tool while preserving or even improving on the previously best solutions.}}, author = {{Mohr, Felix and Wever, Marcel Dominik and Tornede, Alexander and Hüllermeier, Eyke}}, journal = {{IEEE Transactions on Pattern Analysis and Machine Intelligence}}, publisher = {{IEEE}}, title = {{{Predicting Machine Learning Pipeline Runtimes in the Context of Automated Machine Learning}}}, year = {{2021}}, } @article{21535, author = {{Bengs, Viktor and Busa-Fekete, Róbert and El Mesaoudi-Paul, Adil and Hüllermeier, Eyke}}, journal = {{Journal of Machine Learning Research}}, number = {{7}}, pages = {{1--108}}, title = {{{Preference-based Online Learning with Dueling Bandits: A Survey}}}, volume = {{22}}, year = {{2021}}, } @inproceedings{21570, author = {{Tornede, Tanja and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the Genetic and Evolutionary Computation Conference}}, title = {{{Coevolution of Remaining Useful Lifetime Estimation Pipelines for Automated Predictive Maintenance}}}, year = {{2021}}, } @inproceedings{23779, abstract = {{Produktentstehung (PE) bezieht sich auf den Prozess der Planung und Entwicklung eines Produkts sowie der damit verbundenen Dienstleistungen von der ersten Idee bis zur Herstellung und zum Vertrieb. Während dieses Prozesses gibt es zahlreiche Aufgaben, die von menschlichem Fachwissen abhängen und typischerweise von erfahrenen Experten übernommen werden. Da sich das Feld der Künstlichen Intelligenz (KI) immer weiterentwickelt und seinen Weg in den Fertigungssektor findet, gibt es viele Möglichkeiten für eine Anwendung von KI, um bei der Lösung der oben genannten Aufgaben zu helfen. In diesem Paper geben wir einen umfassenden Überblick über den aktuellen Stand der Technik des Einsatzes von KI in der PE. Im Detail analysieren wir 40 bestehende Surveys zu KI in der PE und 94 Case Studies, um herauszufinden, welche Bereiche der PE von der aktuellen Forschung in diesem Bereich vorrangig adressiert werden, wie ausgereift die diskutierten KI-Methoden sind und inwieweit datenzentrierte Ansätze in der aktuellen Forschung genutzt werden.}}, author = {{Bernijazov, Ruslan and Dicks, Alexander and Dumitrescu, Roman and Foullois, Marc and Hanselle, Jonas Manuel and Hüllermeier, Eyke and Karakaya, Gökce and Ködding, Patrick and Lohweg, Volker and Malatyali, Manuel and Meyer auf der Heide, Friedhelm and Panzner, Melina and Soltenborn, Christian}}, booktitle = {{Proceedings of the 30th International Joint Conference on Artificial Intelligence (IJCAI-21)}}, keywords = {{Artificial Intelligence Product Creation Literature Review}}, location = {{Montreal, Kanada}}, title = {{{A Meta-Review on Artificial Intelligence in Product Creation}}}, year = {{2021}}, } @inproceedings{22913, author = {{Hüllermeier, Eyke and Mohr, Felix and Tornede, Alexander and Wever, Marcel Dominik}}, location = {{Bilbao (Virtual)}}, title = {{{Automated Machine Learning, Bounded Rationality, and Rational Metareasoning}}}, year = {{2021}}, } @inproceedings{22914, author = {{Mohr, Felix and Wever, Marcel Dominik}}, location = {{Virtual}}, title = {{{Replacing the Ex-Def Baseline in AutoML by Naive AutoML}}}, year = {{2021}}, } @inproceedings{27381, abstract = {{Graph neural networks (GNNs) have been successfully applied in many structured data domains, with applications ranging from molecular property prediction to the analysis of social networks. Motivated by the broad applicability of GNNs, we propose the family of so-called RankGNNs, a combination of neural Learning to Rank (LtR) methods and GNNs. RankGNNs are trained with a set of pair-wise preferences between graphs, suggesting that one of them is preferred over the other. One practical application of this problem is drug screening, where an expert wants to find the most promising molecules in a large collection of drug candidates. We empirically demonstrate that our proposed pair-wise RankGNN approach either significantly outperforms or at least matches the ranking performance of the naive point-wise baseline approach, in which the LtR problem is solved via GNN-based graph regression.}}, author = {{Damke, Clemens and Hüllermeier, Eyke}}, booktitle = {{Proceedings of The 24th International Conference on Discovery Science (DS 2021)}}, editor = {{Soares, Carlos and Torgo, Luis}}, isbn = {{9783030889418}}, issn = {{0302-9743}}, keywords = {{Graph-structured data, Graph neural networks, Preference learning, Learning to rank}}, location = {{Halifax, Canada}}, pages = {{166--180}}, publisher = {{Springer}}, title = {{{Ranking Structured Objects with Graph Neural Networks}}}, doi = {{10.1007/978-3-030-88942-5}}, volume = {{12986}}, year = {{2021}}, } @phdthesis{27284, author = {{Wever, Marcel Dominik}}, title = {{{Automated Machine Learning for Multi-Label Classification}}}, doi = {{10.17619/UNIPB/1-1302}}, year = {{2021}}, } @inproceedings{21198, author = {{Hanselle, Jonas Manuel and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, location = {{Delhi, India}}, title = {{{Algorithm Selection as Superset Learning: Constructing Algorithm Selectors from Imprecise Performance Data}}}, year = {{2021}}, } @inbook{19521, author = {{Pfannschmidt, Karlson and Hüllermeier, Eyke}}, booktitle = {{Lecture Notes in Computer Science}}, isbn = {{9783030582845}}, issn = {{0302-9743}}, title = {{{Learning Choice Functions via Pareto-Embeddings}}}, doi = {{10.1007/978-3-030-58285-2_30}}, year = {{2020}}, } @inproceedings{19953, abstract = {{Current GNN architectures use a vertex neighborhood aggregation scheme, which limits their discriminative power to that of the 1-dimensional Weisfeiler-Lehman (WL) graph isomorphism test. Here, we propose a novel graph convolution operator that is based on the 2-dimensional WL test. We formally show that the resulting 2-WL-GNN architecture is more discriminative than existing GNN approaches. This theoretical result is complemented by experimental studies using synthetic and real data. On multiple common graph classification benchmarks, we demonstrate that the proposed model is competitive with state-of-the-art graph kernels and GNNs.}}, author = {{Damke, Clemens and Melnikov, Vitaly and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the 12th Asian Conference on Machine Learning (ACML 2020)}}, editor = {{Jialin Pan, Sinno and Sugiyama, Masashi}}, keywords = {{graph neural networks, Weisfeiler-Lehman test, cycle detection}}, location = {{Bangkok, Thailand}}, pages = {{49--64}}, publisher = {{PMLR}}, title = {{{A Novel Higher-order Weisfeiler-Lehman Graph Convolution}}}, volume = {{129}}, year = {{2020}}, } @inproceedings{21534, author = {{Bengs, Viktor and Hüllermeier, Eyke}}, booktitle = {{International Conference on Machine Learning}}, pages = {{778--787}}, title = {{{Preselection Bandits}}}, year = {{2020}}, } @unpublished{21536, abstract = {{We consider a resource-aware variant of the classical multi-armed bandit problem: In each round, the learner selects an arm and determines a resource limit. It then observes a corresponding (random) reward, provided the (random) amount of consumed resources remains below the limit. Otherwise, the observation is censored, i.e., no reward is obtained. For this problem setting, we introduce a measure of regret, which incorporates the actual amount of allocated resources of each learning round as well as the optimality of realizable rewards. Thus, to minimize regret, the learner needs to set a resource limit and choose an arm in such a way that the chance to realize a high reward within the predefined resource limit is high, while the resource limit itself should be kept as low as possible. We derive the theoretical lower bound on the cumulative regret and propose a learning algorithm having a regret upper bound that matches the lower bound. In a simulation study, we show that our learning algorithm outperforms straightforward extensions of standard multi-armed bandit algorithms.}}, author = {{Bengs, Viktor and Hüllermeier, Eyke}}, booktitle = {{arXiv:2011.00813}}, title = {{{Multi-Armed Bandits with Censored Consumption of Resources}}}, year = {{2020}}, } @inproceedings{17407, author = {{Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Discovery Science}}, title = {{{Extreme Algorithm Selection with Dyadic Feature Representation}}}, year = {{2020}}, } @inproceedings{17408, author = {{Hanselle, Jonas Manuel and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{KI 2020: Advances in Artificial Intelligence}}, title = {{{Hybrid Ranking and Regression for Algorithm Selection}}}, year = {{2020}}, } @inproceedings{17424, author = {{Tornede, Tanja and Tornede, Alexander and Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the ECMLPKDD 2020}}, title = {{{AutoML for Predictive Maintenance: One Tool to RUL Them All}}}, doi = {{10.1007/978-3-030-66770-2_8}}, year = {{2020}}, } @unpublished{17605, abstract = {{Syntactic annotation of corpora in the form of part-of-speech (POS) tags is a key requirement for both linguistic research and subsequent automated natural language processing (NLP) tasks. This problem is commonly tackled using machine learning methods, i.e., by training a POS tagger on a sufficiently large corpus of labeled data. While the problem of POS tagging can essentially be considered as solved for modern languages, historical corpora turn out to be much more difficult, especially due to the lack of native speakers and sparsity of training data. Moreover, most texts have no sentences as we know them today, nor a common orthography. These irregularities render the task of automated POS tagging more difficult and error-prone. Under these circumstances, instead of forcing the POS tagger to predict and commit to a single tag, it should be enabled to express its uncertainty. In this paper, we consider POS tagging within the framework of set-valued prediction, which allows the POS tagger to express its uncertainty via predicting a set of candidate POS tags instead of guessing a single one. The goal is to guarantee a high confidence that the correct POS tag is included while keeping the number of candidates small. In our experimental study, we find that extending state-of-the-art POS taggers to set-valued prediction yields more precise and robust taggings, especially for unknown words, i.e., words not occurring in the training data.}}, author = {{Heid, Stefan Helmut and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Journal of Data Mining and Digital Humanities}}, publisher = {{episciences}}, title = {{{Reliable Part-of-Speech Tagging of Historical Corpora through Set-Valued Prediction}}}, year = {{2020}}, } @inproceedings{20306, author = {{Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Workshop MetaLearn 2020 @ NeurIPS 2020}}, location = {{Online}}, title = {{{Towards Meta-Algorithm Selection}}}, year = {{2020}}, } @inbook{18014, author = {{El Mesaoudi-Paul, Adil and Weiß, Dimitri and Bengs, Viktor and Hüllermeier, Eyke and Tierney, Kevin}}, booktitle = {{Learning and Intelligent Optimization. LION 2020.}}, isbn = {{9783030535513}}, issn = {{0302-9743}}, pages = {{216 -- 232}}, publisher = {{Springer}}, title = {{{Pool-Based Realtime Algorithm Configuration: A Preselection Bandit Approach}}}, doi = {{10.1007/978-3-030-53552-0_22}}, volume = {{12096}}, year = {{2020}}, } @unpublished{18017, abstract = {{We consider an extension of the contextual multi-armed bandit problem, in which, instead of selecting a single alternative (arm), a learner is supposed to make a preselection in the form of a subset of alternatives. More specifically, in each iteration, the learner is presented a set of arms and a context, both described in terms of feature vectors. The task of the learner is to preselect $k$ of these arms, among which a final choice is made in a second step. In our setup, we assume that each arm has a latent (context-dependent) utility, and that feedback on a preselection is produced according to a Plackett-Luce model. We propose the CPPL algorithm, which is inspired by the well-known UCB algorithm, and evaluate this algorithm on synthetic and real data. In particular, we consider an online algorithm selection scenario, which served as a main motivation of our problem setting. Here, an instance (which defines the context) from a certain problem class (such as SAT) can be solved by different algorithms (the arms), but only $k$ of these algorithms can actually be run.}}, author = {{El Mesaoudi-Paul, Adil and Bengs, Viktor and Hüllermeier, Eyke}}, booktitle = {{arXiv:2002.04275}}, title = {{{Online Preselection with Context Information under the Plackett-Luce Model}}}, year = {{2020}}, } @inproceedings{18276, abstract = {{Algorithm selection (AS) deals with the automatic selection of an algorithm from a fixed set of candidate algorithms most suitable for a specific instance of an algorithmic problem class, where "suitability" often refers to an algorithm's runtime. Due to possibly extremely long runtimes of candidate algorithms, training data for algorithm selection models is usually generated under time constraints in the sense that not all algorithms are run to completion on all instances. Thus, training data usually comprises censored information, as the true runtime of algorithms timed out remains unknown. However, many standard AS approaches are not able to handle such information in a proper way. On the other side, survival analysis (SA) naturally supports censored data and offers appropriate ways to use such data for learning distributional models of algorithm runtime, as we demonstrate in this work. We leverage such models as a basis of a sophisticated decision-theoretic approach to algorithm selection, which we dub Run2Survive. Moreover, taking advantage of a framework of this kind, we advocate a risk-averse approach to algorithm selection, in which the avoidance of a timeout is given high priority. In an extensive experimental study with the standard benchmark ASlib, our approach is shown to be highly competitive and in many cases even superior to state-of-the-art AS approaches.}}, author = {{Tornede, Alexander and Wever, Marcel Dominik and Werner, Stefan and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{ACML 2020}}, location = {{Bangkok, Thailand}}, title = {{{Run2Survive: A Decision-theoretic Approach to Algorithm Selection based on Survival Analysis}}}, year = {{2020}}, } @article{16725, author = {{Richter, Cedric and Hüllermeier, Eyke and Jakobs, Marie-Christine and Wehrheim, Heike}}, journal = {{Journal of Automated Software Engineering}}, publisher = {{Springer}}, title = {{{Algorithm Selection for Software Validation Based on Graph Kernels}}}, year = {{2020}}, } @inproceedings{15629, abstract = {{In multi-label classification (MLC), each instance is associated with a set of class labels, in contrast to standard classification where an instance is assigned a single label. Binary relevance (BR) learning, which reduces a multi-label to a set of binary classification problems, one per label, is arguably the most straight-forward approach to MLC. In spite of its simplicity, BR proved to be competitive to more sophisticated MLC methods, and still achieves state-of-the-art performance for many loss functions. Somewhat surprisingly, the optimal choice of the base learner for tackling the binary classification problems has received very little attention so far. Taking advantage of the label independence assumption inherent to BR, we propose a label-wise base learner selection method optimizing label-wise macro averaged performance measures. In an extensive experimental evaluation, we find that or approach, called LiBRe, can significantly improve generalization performance.}}, author = {{Wever, Marcel Dominik and Tornede, Alexander and Mohr, Felix and Hüllermeier, Eyke}}, location = {{Konstanz, Germany}}, publisher = {{Springer}}, title = {{{LiBRe: Label-Wise Selection of Base Learners in Binary Relevance for Multi-Label Classification}}}, year = {{2020}}, } @article{15025, abstract = {{In software engineering, the imprecise requirements of a user are transformed to a formal requirements specification during the requirements elicitation process. This process is usually guided by requirements engineers interviewing the user. We want to partially automate this first step of the software engineering process in order to enable users to specify a desired software system on their own. With our approach, users are only asked to provide exemplary behavioral descriptions. The problem of synthesizing a requirements specification from examples can partially be reduced to the problem of grammatical inference, to which we apply an active coevolutionary learning approach. However, this approach would usually require many feedback queries to be sent to the user. In this work, we extend and generalize our active learning approach to receive knowledge from multiple oracles, also known as proactive learning. The ‘user oracle’ represents input received from the user and the ‘knowledge oracle’ represents available, formalized domain knowledge. We call our two-oracle approach the ‘first apply knowledge then query’ (FAKT/Q) algorithm. We compare FAKT/Q to the active learning approach and provide an extensive benchmark evaluation. As result we find that the number of required user queries is reduced and the inference process is sped up significantly. Finally, with so-called On-The-Fly Markets, we present a motivation and an application of our approach where such knowledge is available.}}, author = {{Wever, Marcel Dominik and van Rooijen, Lorijn and Hamann, Heiko}}, journal = {{Evolutionary Computation}}, number = {{2}}, pages = {{165–193}}, publisher = {{MIT Press Journals}}, title = {{{Multi-Oracle Coevolutionary Learning of Requirements Specifications from Examples in On-The-Fly Markets}}}, doi = {{10.1162/evco_a_00266}}, volume = {{28}}, year = {{2020}}, } @unpublished{19523, abstract = {{We study the problem of learning choice functions, which play an important role in various domains of application, most notably in the field of economics. Formally, a choice function is a mapping from sets to sets: Given a set of choice alternatives as input, a choice function identifies a subset of most preferred elements. Learning choice functions from suitable training data comes with a number of challenges. For example, the sets provided as input and the subsets produced as output can be of any size. Moreover, since the order in which alternatives are presented is irrelevant, a choice function should be symmetric. Perhaps most importantly, choice functions are naturally context-dependent, in the sense that the preference in favor of an alternative may depend on what other options are available. We formalize the problem of learning choice functions and present two general approaches based on two representations of context-dependent utility functions. Both approaches are instantiated by means of appropriate neural network architectures, and their performance is demonstrated on suitable benchmark tasks.}}, author = {{Pfannschmidt, Karlson and Gupta, Pritha and Hüllermeier, Eyke}}, booktitle = {{arXiv:1901.10860}}, title = {{{Learning Choice Functions: Concepts and Architectures}}}, year = {{2019}}, } @article{17565, author = {{Merten, Marie-Luis and Seemann, Nina and Wever, Marcel Dominik}}, journal = {{Niederdeutsches Jahrbuch}}, number = {{142}}, pages = {{124--146}}, title = {{{Grammatikwandel digital-kulturwissenschaftlich erforscht. Mittelniederdeutscher Sprachausbau im interdisziplinären Zugriff}}}, year = {{2019}}, } @unpublished{18018, abstract = {{A common statistical task lies in showing asymptotic normality of certain statistics. In many of these situations, classical textbook results on weak convergence theory suffice for the problem at hand. However, there are quite some scenarios where stronger results are needed in order to establish an asymptotic normal approximation uniformly over a family of probability measures. In this note we collect some results in this direction. We restrict ourselves to weak convergence in $\mathbb R^d$ with continuous limit measures.}}, author = {{Bengs, Viktor and Holzmann, Hajo}}, booktitle = {{arXiv:1903.09864}}, title = {{{Uniform approximation in classical weak convergence theory}}}, year = {{2019}}, } @inproceedings{8868, author = {{Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke and Hetzer, Alexander}}, location = {{Bayreuth, Germany}}, title = {{{Towards Automated Machine Learning for Multi-Label Classification}}}, year = {{2019}}, } @article{10578, author = {{Tagne, V. K. and Fotso, S. and Fono, L. A. and Hüllermeier, Eyke}}, journal = {{New Mathematics and Natural Computation}}, number = {{2}}, pages = {{191--213}}, title = {{{Choice Functions Generated by Mallows and Plackett–Luce Relations}}}, volume = {{15}}, year = {{2019}}, } @article{15001, author = {{Couso, Ines and Borgelt, Christian and Hüllermeier, Eyke and Kruse, Rudolf}}, issn = {{1556-603X}}, journal = {{IEEE Computational Intelligence Magazine}}, pages = {{31--44}}, title = {{{Fuzzy Sets in Data Analysis: From Statistical Foundations to Machine Learning}}}, doi = {{10.1109/mci.2018.2881642}}, year = {{2019}}, } @article{15002, abstract = {{Many problem settings in machine learning are concerned with the simultaneous prediction of multiple target variables of diverse type. Amongst others, such problem settings arise in multivariate regression, multi-label classification, multi-task learning, dyadic prediction, zero-shot learning, network inference, and matrix completion. These subfields of machine learning are typically studied in isolation, without highlighting or exploring important relationships. In this paper, we present a unifying view on what we call multi-target prediction (MTP) problems and methods. First, we formally discuss commonalities and differences between existing MTP problems. To this end, we introduce a general framework that covers the above subfields as special cases. As a second contribution, we provide a structured overview of MTP methods. This is accomplished by identifying a number of key properties, which distinguish such methods and determine their suitability for different types of problems. Finally, we also discuss a few challenges for future research.}}, author = {{Waegeman, Willem and Dembczynski, Krzysztof and Hüllermeier, Eyke}}, issn = {{1573-756X}}, journal = {{Data Mining and Knowledge Discovery}}, number = {{2}}, pages = {{293--324}}, title = {{{Multi-target prediction: a unifying view on problems and methods}}}, doi = {{10.1007/s10618-018-0595-5}}, volume = {{33}}, year = {{2019}}, } @inproceedings{15003, author = {{Mortier, Thomas and Wydmuch, Marek and Dembczynski, Krzysztof and Hüllermeier, Eyke and Waegeman, Willem}}, booktitle = {{Proceedings of the 31st Benelux Conference on Artificial Intelligence {(BNAIC} 2019) and the 28th Belgian Dutch Conference on Machine Learning (Benelearn 2019), Brussels, Belgium, November 6-8, 2019}}, title = {{{Set-Valued Prediction in Multi-Class Classification}}}, year = {{2019}}, } @inbook{15004, author = {{Ahmadi Fahandar, Mohsen and Hüllermeier, Eyke}}, booktitle = {{Discovery Science}}, isbn = {{9783030337773}}, issn = {{0302-9743}}, title = {{{Feature Selection for Analogy-Based Learning to Rank}}}, doi = {{10.1007/978-3-030-33778-0_22}}, year = {{2019}}, } @inbook{15005, author = {{Ahmadi Fahandar, Mohsen and Hüllermeier, Eyke}}, booktitle = {{KI 2019: Advances in Artificial Intelligence}}, isbn = {{9783030301781}}, issn = {{0302-9743}}, title = {{{Analogy-Based Preference Learning with Kernels}}}, doi = {{10.1007/978-3-030-30179-8_3}}, year = {{2019}}, } @inbook{15006, author = {{Nguyen, Vu-Linh and Destercke, Sébastien and Hüllermeier, Eyke}}, booktitle = {{Discovery Science}}, isbn = {{9783030337773}}, issn = {{0302-9743}}, title = {{{Epistemic Uncertainty Sampling}}}, doi = {{10.1007/978-3-030-33778-0_7}}, year = {{2019}}, } @inproceedings{15007, author = {{Melnikov, Vitaly and Hüllermeier, Eyke}}, booktitle = {{Proceedings ACML, Asian Conference on Machine Learning (Proceedings of Machine Learning Research, 101)}}, title = {{{Learning to Aggregate: Tackling the Aggregation/Disaggregation Problem for OWA}}}, doi = {{10.1016/j.jmva.2019.02.017}}, year = {{2019}}, } @inproceedings{15009, author = {{Epple, Nico and Dari, Simone and Drees, Ludwig and Protschky, Valentin and Riener, Andreas}}, booktitle = {{2019 IEEE Intelligent Vehicles Symposium (IV)}}, isbn = {{9781728105604}}, title = {{{Influence of Cruise Control on Driver Guidance - a Comparison between System Generations and Countries}}}, doi = {{10.1109/ivs.2019.8814100}}, year = {{2019}}, } @inproceedings{15011, author = {{Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Proceedings - 29. Workshop Computational Intelligence, Dortmund, 28. - 29. November 2019}}, editor = {{Hoffmann, Frank and Hüllermeier, Eyke and Mikut, Ralf}}, isbn = {{978-3-7315-0979-0}}, location = {{Dortmund}}, pages = {{135--146}}, publisher = {{KIT Scientific Publishing, Karlsruhe}}, title = {{{Algorithm Selection as Recommendation: From Collaborative Filtering to Dyad Ranking}}}, year = {{2019}}, } @inproceedings{15013, author = {{Brinker, Klaus and Hüllermeier, Eyke}}, booktitle = {{Proceedings ECML/PKDD, European Conference on Machine Learning and Knowledge Discovery in Databases}}, title = {{{A Reduction of Label Ranking to Multiclass Classification}}}, year = {{2019}}, } @inproceedings{15014, author = {{Hüllermeier, Eyke and Couso, Ines and Diestercke, Sebastian}}, booktitle = {{Proceedings SUM 2019, International Conference on Scalable Uncertainty Management}}, title = {{{Learning from Imprecise Data: Adjustments of Optimistic and Pessimistic Variants}}}, year = {{2019}}, } @article{15015, author = {{Henzgen, Sascha and Hüllermeier, Eyke}}, issn = {{1556-4681}}, journal = {{ACM Transactions on Knowledge Discovery from Data}}, pages = {{1--36}}, title = {{{Mining Rank Data}}}, doi = {{10.1145/3363572}}, year = {{2019}}, } @article{14027, author = {{Bengs, Viktor and Eulert, Matthias and Holzmann, Hajo}}, issn = {{0047-259X}}, journal = {{Journal of Multivariate Analysis}}, pages = {{291--312}}, title = {{{Asymptotic confidence sets for the jump curve in bivariate regression problems}}}, doi = {{10.1016/j.jmva.2019.02.017}}, year = {{2019}}, } @article{14028, author = {{Bengs, Viktor and Holzmann, Hajo}}, issn = {{1935-7524}}, journal = {{Electronic Journal of Statistics}}, pages = {{1523--1579}}, title = {{{Adaptive confidence sets for kink estimation}}}, doi = {{10.1214/19-ejs1555}}, year = {{2019}}, } @inproceedings{13132, author = {{Mohr, Felix and Wever, Marcel Dominik and Tornede, Alexander and Hüllermeier, Eyke}}, booktitle = {{INFORMATIK 2019: 50 Jahre Gesellschaft für Informatik – Informatik für Gesellschaft}}, location = {{Kassel}}, pages = {{ 273--274 }}, publisher = {{Gesellschaft für Informatik e.V.}}, title = {{{From Automated to On-The-Fly Machine Learning}}}, year = {{2019}}, } @inproceedings{10232, abstract = {{Existing tools for automated machine learning, such as Auto-WEKA, TPOT, auto-sklearn, and more recently ML-Plan, have shown impressive results for the tasks of single-label classification and regression. Yet, there is only little work on other types of machine learning problems so far. In particular, there is almost no work on automating the engineering of machine learning solutions for multi-label classification (MLC). We show how the scope of ML-Plan, an AutoML-tool for multi-class classification, can be extended towards MLC using MEKA, which is a multi-label extension of the well-known Java library WEKA. The resulting approach recursively refines MEKA's multi-label classifiers, nesting other multi-label classifiers for meta algorithms and single-label classifiers provided by WEKA as base learners. In our evaluation, we find that the proposed approach yields strong results and performs significantly better than a set of baselines we compare with.}}, author = {{Wever, Marcel Dominik and Mohr, Felix and Tornede, Alexander and Hüllermeier, Eyke}}, location = {{Long Beach, CA, USA}}, title = {{{Automating Multi-Label Classification Extending ML-Plan}}}, year = {{2019}}, } @article{20243, author = {{Rohlfing, Katharina and Leonardi, Giuseppe and Nomikou, Iris and Rączaszek-Leonardi, Joanna and Hüllermeier, Eyke}}, journal = {{IEEE Transactions on Cognitive and Developmental Systems}}, title = {{{Multimodal Turn-Taking: Motivations, Methodological Challenges, and Novel Approaches}}}, doi = {{10.1109/TCDS.2019.2892991}}, year = {{2019}}, } @inproceedings{2479, author = {{Mohr, Felix and Wever, Marcel Dominik and Hüllermeier, Eyke and Faez, Amin}}, booktitle = {{SCC}}, location = {{San Francisco, CA, USA}}, publisher = {{IEEE}}, title = {{{(WIP) Towards the Automated Composition of Machine Learning Services}}}, doi = {{10.1109/SCC.2018.00039}}, year = {{2018}}, } @unpublished{19524, abstract = {{Object ranking is an important problem in the realm of preference learning. On the basis of training data in the form of a set of rankings of objects, which are typically represented as feature vectors, the goal is to learn a ranking function that predicts a linear order of any new set of objects. Current approaches commonly focus on ranking by scoring, i.e., on learning an underlying latent utility function that seeks to capture the inherent utility of each object. These approaches, however, are not able to take possible effects of context-dependence into account, where context-dependence means that the utility or usefulness of an object may also depend on what other objects are available as alternatives. In this paper, we formalize the problem of context-dependent ranking and present two general approaches based on two natural representations of context-dependent ranking functions. Both approaches are instantiated by means of appropriate neural network architectures, which are evaluated on suitable benchmark task.}}, author = {{Pfannschmidt, Karlson and Gupta, Pritha and Hüllermeier, Eyke}}, booktitle = {{arXiv:1803.05796}}, title = {{{Deep Architectures for Learning Context-dependent Ranking Functions}}}, year = {{2018}}, } @inproceedings{2857, author = {{Mohr, Felix and Lettmann, Theodor and Hüllermeier, Eyke and Wever, Marcel Dominik}}, booktitle = {{Proceedings of the 1st ICAPS Workshop on Hierarchical Planning}}, location = {{Delft, Netherlands}}, pages = {{31--39}}, publisher = {{AAAI}}, title = {{{Programmatic Task Network Planning}}}, year = {{2018}}, } @article{24150, author = {{Ramaswamy, Arunselvan and Bhatnagar, Shalabh}}, journal = {{IEEE Transactions on Automatic Control}}, number = {{6}}, pages = {{2614--2620}}, publisher = {{IEEE}}, title = {{{Stability of stochastic approximations with “controlled markov” noise and temporal difference learning}}}, volume = {{64}}, year = {{2018}}, } @article{24151, author = {{Demirel, Burak and Ramaswamy, Arunselvan and Quevedo, Daniel E and Karl, Holger}}, journal = {{IEEE Control Systems Letters}}, number = {{4}}, pages = {{737--742}}, publisher = {{IEEE}}, title = {{{Deepcas: A deep reinforcement learning algorithm for control-aware scheduling}}}, volume = {{2}}, year = {{2018}}, } @inproceedings{2471, author = {{Mohr, Felix and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{SCC}}, location = {{San Francisco, CA, USA}}, publisher = {{IEEE Computer Society}}, title = {{{On-The-Fly Service Construction with Prototypes}}}, doi = {{10.1109/SCC.2018.00036}}, year = {{2018}}, } @article{3402, abstract = {{In machine learning, so-called nested dichotomies are utilized as a reduction technique, i.e., to decompose a multi-class classification problem into a set of binary problems, which are solved using a simple binary classifier as a base learner. The performance of the (multi-class) classifier thus produced strongly depends on the structure of the decomposition. In this paper, we conduct an empirical study, in which we compare existing heuristics for selecting a suitable structure in the form of a nested dichotomy. Moreover, we propose two additional heuristics as natural completions. One of them is the Best-of-K heuristic, which picks the (presumably) best among K randomly generated nested dichotomies. Surprisingly, and in spite of its simplicity, it turns out to outperform the state of the art.}}, author = {{Melnikov, Vitalik and Hüllermeier, Eyke}}, issn = {{1573-0565}}, journal = {{Machine Learning}}, title = {{{On the effectiveness of heuristics for learning nested dichotomies: an empirical analysis}}}, doi = {{10.1007/s10994-018-5733-1}}, year = {{2018}}, } @article{3510, abstract = {{Automated machine learning (AutoML) seeks to automatically select, compose, and parametrize machine learning algorithms, so as to achieve optimal performance on a given task (dataset). Although current approaches to AutoML have already produced impressive results, the field is still far from mature, and new techniques are still being developed. In this paper, we present ML-Plan, a new approach to AutoML based on hierarchical planning. To highlight the potential of this approach, we compare ML-Plan to the state-of-the-art frameworks Auto-WEKA, auto-sklearn, and TPOT. In an extensive series of experiments, we show that ML-Plan is highly competitive and often outperforms existing approaches.}}, author = {{Mohr, Felix and Wever, Marcel Dominik and Hüllermeier, Eyke}}, issn = {{1573-0565}}, journal = {{Machine Learning}}, keywords = {{AutoML, Hierarchical Planning, HTN planning, ML-Plan}}, location = {{Dublin, Ireland}}, pages = {{1495--1515}}, publisher = {{Springer}}, title = {{{ML-Plan: Automated Machine Learning via Hierarchical Planning}}}, doi = {{10.1007/s10994-018-5735-z}}, year = {{2018}}, } @inproceedings{3552, author = {{Mohr, Felix and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the Symposium on Intelligent Data Analysis}}, location = {{‘s-Hertogenbosch, the Netherlands}}, title = {{{Reduction Stumps for Multi-Class Classification}}}, doi = {{10.1007/978-3-030-01768-2_19}}, year = {{2018}}, } @inproceedings{3852, abstract = {{In automated machine learning (AutoML), the process of engineering machine learning applications with respect to a specific problem is (partially) automated. Various AutoML tools have already been introduced to provide out-of-the-box machine learning functionality. More specifically, by selecting machine learning algorithms and optimizing their hyperparameters, these tools produce a machine learning pipeline tailored to the problem at hand. Except for TPOT, all of these tools restrict the maximum number of processing steps of such a pipeline. However, as TPOT follows an evolutionary approach, it suffers from performance issues when dealing with larger datasets. In this paper, we present an alternative approach leveraging a hierarchical planning to configure machine learning pipelines that are unlimited in length. We evaluate our approach and find its performance to be competitive with other AutoML tools, including TPOT.}}, author = {{Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{ICML 2018 AutoML Workshop}}, keywords = {{automated machine learning, complex pipelines, hierarchical planning}}, location = {{Stockholm, Sweden}}, title = {{{ML-Plan for Unlimited-Length Machine Learning Pipelines}}}, year = {{2018}}, } @inproceedings{2109, abstract = {{In multinomial classification, reduction techniques are commonly used to decompose the original learning problem into several simpler problems. For example, by recursively bisecting the original set of classes, so-called nested dichotomies define a set of binary classification problems that are organized in the structure of a binary tree. In contrast to the existing one-shot heuristics for constructing nested dichotomies and motivated by recent work on algorithm configuration, we propose a genetic algorithm for optimizing the structure of such dichotomies. A key component of this approach is the proposed genetic representation that facilitates the application of standard genetic operators, while still supporting the exchange of partial solutions under recombination. We evaluate the approach in an extensive experimental study, showing that it yields classifiers with superior generalization performance.}}, author = {{Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the Genetic and Evolutionary Computation Conference, GECCO 2018, Kyoto, Japan, July 15-19, 2018}}, keywords = {{Classification, Hierarchical Decomposition, Indirect Encoding}}, location = {{Kyoto, Japan}}, publisher = {{ACM}}, title = {{{Ensembles of Evolved Nested Dichotomies for Classification}}}, doi = {{10.1145/3205455.3205562}}, year = {{2018}}, } @unpublished{17713, author = {{Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke}}, publisher = {{Arxiv}}, title = {{{Automated Multi-Label Classification based on ML-Plan}}}, year = {{2018}}, } @unpublished{17714, author = {{Mohr, Felix and Wever, Marcel Dominik and Hüllermeier, Eyke}}, title = {{{Automated machine learning service composition}}}, year = {{2018}}, } @misc{5693, author = {{Graf, Helena}}, publisher = {{Universität Paderborn}}, title = {{{Ranking of Classification Algorithms in AutoML}}}, year = {{2018}}, } @misc{5936, author = {{Scheibl, Manuel}}, publisher = {{Universität Paderborn}}, title = {{{Learning about learning curves from dataset properties}}}, year = {{2018}}, } @inbook{6423, author = {{Schäfer, Dirk and Hüllermeier, Eyke}}, booktitle = {{Discovery Science}}, isbn = {{9783030017705}}, issn = {{0302-9743}}, pages = {{161--175}}, publisher = {{Springer International Publishing}}, title = {{{Preference-Based Reinforcement Learning Using Dyad Ranking}}}, doi = {{10.1007/978-3-030-01771-2_11}}, year = {{2018}}, } @proceedings{10591, editor = {{Abiteboul, S. and Arenas, M. and Barceló, P. and Bienvenu, M. and Calvanese, D. and David, C. and Hull, R. and Hüllermeier, Eyke and Kimelfeld, B. and Libkin, L. and Martens, W. and Milo, T. and Murlak, F. and Neven, F. and Ortiz, M. and Schwentick, T. and Stoyanovich, J. and Su, J. and Suciu, D. and Vianu, V. and Yi, K.}}, number = {{1}}, pages = {{1--29}}, title = {{{Research Directions for Principles of Data Management}}}, volume = {{7}}, year = {{2018}}, } @inbook{10783, author = {{Couso, Ines and Hüllermeier, Eyke}}, booktitle = {{Frontiers in Computational Intelligence}}, editor = {{Mostaghim, Sanaz and Nürnberger, Andreas and Borgelt, Christian}}, pages = {{31--46}}, publisher = {{Springer}}, title = {{{Statistical Inference for Incomplete Ranking Data: A Comparison of two likelihood-based estimators}}}, year = {{2018}}, } @article{16038, author = {{Schäfer, D. and Hüllermeier, Eyke}}, journal = {{Machine Learning}}, number = {{5}}, pages = {{903--941}}, title = {{{Dyad ranking using Plackett-Luce models based on joint feature representations}}}, volume = {{107}}, year = {{2018}}, } @inproceedings{10145, author = {{Ahmadi Fahandar, Mohsen and Hüllermeier, Eyke}}, booktitle = {{Proc. 32 nd AAAI Conference on Artificial Intelligence (AAAI)}}, pages = {{2951--2958}}, title = {{{Learning to Rank Based on Analogical Reasoning}}}, year = {{2018}}, } @inproceedings{10148, author = {{El Mesaoudi-Paul, Adil and Hüllermeier, Eyke and Busa-Fekete, Robert}}, booktitle = {{Proc. 35th Int. Conference on Machine Learning (ICML)}}, pages = {{3469--3477}}, publisher = {{Verlagsschriftenreihe des Heinz Nixdorf Instituts, Paderborn}}, title = {{{Ranking Distributions based on Noisy Sorting}}}, year = {{2018}}, } @inproceedings{10149, author = {{Hesse, M. and Timmermann, J. and Hüllermeier, Eyke and Trächtler, Ansgar}}, booktitle = {{Proc. 4th Int. Conference on System-Integrated Intelligence: Intelligent, Flexible and Connected Systems in Products and Production, Procedia Manufacturing 24}}, pages = {{15--20}}, title = {{{A Reinforcement Learning Strategy for the Swing-Up of the Double Pendulum on a Cart}}}, year = {{2018}}, } @inbook{10152, author = {{Mencia, E.Loza and Fürnkranz, J. and Hüllermeier, Eyke and Rapp, M.}}, booktitle = {{Explainable and Interpretable Models in Computer Vision and Machine Learning}}, editor = {{Jair Escalante, H. and Escalera, S. and Guyon, I. and Baro, X. and Güclüütürk, Y. and Güclü, U. and van Gerven, M.A.J.}}, pages = {{81--113}}, publisher = {{Springer}}, title = {{{Learning interpretable rules for multi-label classification}}}, year = {{2018}}, } @inproceedings{10181, author = {{Nguyen, Vu-Linh and Destercke, Sebastian and Masson, M.-H. and Hüllermeier, Eyke}}, booktitle = {{Proc. 27th Int.Joint Conference on Artificial Intelligence (IJCAI)}}, pages = {{5089--5095}}, title = {{{Reliable Multi-class Classification based on Pairwise Epistemic and Aleatoric Uncertainty}}}, year = {{2018}}, } @inproceedings{10184, author = {{Schäfer, Dirk and Hüllermeier, Eyke}}, booktitle = {{Proc. 21st Int. Conference on Discovery Science (DS)}}, pages = {{161--175}}, title = {{{Preference-Based Reinforcement Learning Using Dyad Ranking}}}, year = {{2018}}, } @article{10276, author = {{Schäfer, Dirk and Hüllermeier, Eyke}}, journal = {{Machine Learning}}, number = {{5}}, pages = {{903--941}}, title = {{{Dyad Ranking Using Plackett-Luce Models based on joint feature representations}}}, volume = {{107}}, year = {{2018}}, } @inproceedings{1379, author = {{Seemann, Nina and Geierhos, Michaela and Merten, Marie-Luis and Tophinke, Doris and Wever, Marcel Dominik and Hüllermeier, Eyke}}, booktitle = {{Postersession Computerlinguistik der 40. Jahrestagung der Deutschen Gesellschaft für Sprachwissenschaft}}, editor = {{Eckart, Kerstin and Schlechtweg, Dominik }}, location = {{Stuttgart, Germany}}, title = {{{Supporting the Cognitive Process in Annotation Tasks}}}, year = {{2018}}, } @article{24152, author = {{Ramaswamy, Arunselvan and Bhatnagar, Shalabh}}, journal = {{IEEE Transactions on Automatic Control}}, number = {{5}}, pages = {{1465--1471}}, publisher = {{IEEE}}, title = {{{Analysis of gradient descent methods with nondiminishing bounded errors}}}, volume = {{63}}, year = {{2017}}, } @article{24153, author = {{Ramaswamy, Arunselvan and Bhatnagar, Shalabh}}, journal = {{Mathematics of Operations Research}}, number = {{3}}, pages = {{648--661}}, publisher = {{INFORMS}}, title = {{{A generalization of the Borkar-Meyn theorem for stochastic recursive inclusions}}}, volume = {{42}}, year = {{2017}}, } @inproceedings{3325, author = {{Melnikov, Vitalik and Hüllermeier, Eyke}}, booktitle = {{Proceedings. 27. Workshop Computational Intelligence, Dortmund, 23. - 24. November 2017}}, publisher = {{KIT Scientific Publishing}}, title = {{{Optimizing the Structure of Nested Dichotomies: A Comparison of Two Heuristics}}}, doi = {{10.5445/KSP/1000074341}}, year = {{2017}}, } @inproceedings{115, abstract = {{Whenever customers have to decide between different instances of the same product, they are interested in buying the best product. In contrast, companies are interested in reducing the construction effort (and usually as a consequence thereof, the quality) to gain profit. The described setting is widely known as opposed preferences in quality of the product and also applies to the context of service-oriented computing. In general, service-oriented computing emphasizes the construction of large software systems out of existing services, where services are small and self-contained pieces of software that adhere to a specified interface. Several implementations of the same interface are considered as several instances of the same service. Thereby, customers are interested in buying the best service implementation for their service composition wrt. to metrics, such as costs, energy, memory consumption, or execution time. One way to ensure the service quality is to employ certificates, which can come in different kinds: Technical certificates proving correctness can be automatically constructed by the service provider and again be automatically checked by the user. Digital certificates allow proof of the integrity of a product. Other certificates might be rolled out if service providers follow a good software construction principle, which is checked in annual audits. Whereas all of these certificates are handled differently in service markets, what they have in common is that they influence the buying decisions of customers. In this paper, we review state-of-the-art developments in certification with respect to service-oriented computing. We not only discuss how certificates are constructed and handled in service-oriented computing but also review the effects of certificates on the market from an economic perspective.}}, author = {{Jakobs, Marie-Christine and Krämer, Julia and van Straaten, Dirk and Lettmann, Theodor}}, booktitle = {{The Ninth International Conferences on Advanced Service Computing (SERVICE COMPUTATION)}}, editor = {{Marcelo De Barros, Janusz Klink,Tadeus Uhl, Thomas Prinz}}, pages = {{7--12}}, title = {{{Certification Matters for Service Markets}}}, year = {{2017}}, } @inproceedings{1158, abstract = {{In this paper, we present the annotation challenges we have encountered when working on a historical language that was undergoing elaboration processes. We especially focus on syntactic ambiguity and gradience in Middle Low German, which causes uncertainty to some extent. Since current annotation tools consider construction contexts and the dynamics of the grammaticalization only partially, we plan to extend CorA – a web-based annotation tool for historical and other non-standard language data – to capture elaboration phenomena and annotator unsureness. Moreover, we seek to interactively learn morphological as well as syntactic annotations.}}, author = {{Seemann, Nina and Merten, Marie-Luis and Geierhos, Michaela and Tophinke, Doris and Hüllermeier, Eyke}}, booktitle = {{Proceedings of the Joint SIGHUM Workshop on Computational Linguistics for Cultural Heritage, Social Sciences, Humanities and Literature}}, location = {{Vancouver, BC, Canada}}, pages = {{40--45}}, publisher = {{Association for Computational Linguistics (ACL)}}, title = {{{Annotation Challenges for Reconstructing the Structural Elaboration of Middle Low German}}}, doi = {{10.18653/v1/W17-2206}}, year = {{2017}}, } @misc{5694, author = {{Schnitker, Nino Noel}}, publisher = {{Universität Paderborn}}, title = {{{Genetischer Algorithmus zur Erstellung von Ensembles von Nested Dichotomies}}}, year = {{2017}}, } @inproceedings{5722, author = {{Gupta, Pritha and Hetzer, Alexander and Tornede, Tanja and Gottschalk, Sebastian and Kornelsen, Andreas and Osterbrink, Sebastian and Pfannschmidt, Karlson and Hüllermeier, Eyke}}, location = {{Rostock}}, title = {{{jPL: A Java-based Software Framework for Preference Learning}}}, year = {{2017}}, } @misc{5724, author = {{Hetzer, Alexander and Tornede, Tanja}}, publisher = {{Universität Paderborn}}, title = {{{Solving the Container Pre-Marshalling Problem using Reinforcement Learning and Structured Output Prediction}}}, year = {{2017}}, } @inproceedings{71, abstract = {{Today, software verification tools have reached the maturity to be used for large scale programs. Different tools perform differently well on varying code. A software developer is hence faced with the problem of choosing a tool appropriate for her program at hand. A ranking of tools on programs could facilitate the choice. Such rankings can, however, so far only be obtained by running all considered tools on the program.In this paper, we present a machine learning approach to predicting rankings of tools on programs. The method builds upon so-called label ranking algorithms, which we complement with appropriate kernels providing a similarity measure for programs. Our kernels employ a graph representation for software source code that mixes elements of control flow and program dependence graphs with abstract syntax trees. Using data sets from the software verification competition SV-COMP, we demonstrate our rank prediction technique to generalize well and achieve a rather high predictive accuracy (rank correlation > 0.6).}}, author = {{Czech, Mike and Hüllermeier, Eyke and Jakobs, Marie-Christine and Wehrheim, Heike}}, booktitle = {{Proceedings of the 3rd International Workshop on Software Analytics}}, pages = {{23--26}}, title = {{{Predicting Rankings of Software Verification Tools}}}, doi = {{10.1145/3121257.3121262}}, year = {{2017}}, } @techreport{72, abstract = {{Software verification competitions, such as the annual SV-COMP, evaluate software verification tools with respect to their effectivity and efficiency. Typically, the outcome of a competition is a (possibly category-specific) ranking of the tools. For many applications, such as building portfolio solvers, it would be desirable to have an idea of the (relative) performance of verification tools on a given verification task beforehand, i.e., prior to actually running all tools on the task.In this paper, we present a machine learning approach to predicting rankings of tools on verification tasks. The method builds upon so-called label ranking algorithms, which we complement with appropriate kernels providing a similarity measure for verification tasks. Our kernels employ a graph representation for software source code that mixes elements of control flow and program dependence graphs with abstract syntax trees. Using data sets from SV-COMP, we demonstrate our rank prediction technique to generalize well and achieve a rather high predictive accuracy. In particular, our method outperforms a recently proposed feature-based approach of Demyanova et al. (when applied to rank predictions). }}, author = {{Czech, Mike and Hüllermeier, Eyke and Jakobs, Marie-Christine and Wehrheim, Heike}}, title = {{{Predicting Rankings of Software Verification Competitions}}}, year = {{2017}}, } @misc{10589, author = {{Fürnkranz, J. and Hüllermeier, Eyke}}, booktitle = {{Encyclopedia of Machine Learning and Data Mining}}, pages = {{1000--1005}}, title = {{{Preference Learning}}}, year = {{2017}}, } @inbook{10784, author = {{Fürnkranz, J. and Hüllermeier, Eyke}}, booktitle = {{Encyclopedia of Machine Learning and Data Mining}}, editor = {{Sammut, C. and Webb, G.I.}}, pages = {{1000--1005}}, publisher = {{Springer}}, title = {{{Preference Learning}}}, volume = {{107}}, year = {{2017}}, } @inproceedings{1180, abstract = {{These days, there is a strong rise in the needs for machine learning applications, requiring an automation of machine learning engineering which is referred to as AutoML. In AutoML the selection, composition and parametrization of machine learning algorithms is automated and tailored to a specific problem, resulting in a machine learning pipeline. Current approaches reduce the AutoML problem to optimization of hyperparameters. Based on recursive task networks, in this paper we present one approach from the field of automated planning and one evolutionary optimization approach. Instead of simply parametrizing a given pipeline, this allows for structure optimization of machine learning pipelines, as well. We evaluate the two approaches in an extensive evaluation, finding both approaches to have their strengths in different areas. Moreover, the two approaches outperform the state-of-the-art tool Auto-WEKA in many settings.}}, author = {{Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke}}, booktitle = {{27th Workshop Computational Intelligence}}, location = {{Dortmund}}, title = {{{Automatic Machine Learning: Hierachical Planning Versus Evolutionary Optimization}}}, year = {{2017}}, } @inproceedings{15397, author = {{Melnikov, Vitaly and Hüllermeier, Eyke}}, booktitle = {{in Proceedings 27th Workshop Computational Intelligence, Dortmund Germany}}, editor = {{Hoffmann, F. and Hüllermeier, Eyke and Mikut, R.}}, pages = {{1--12}}, publisher = {{KIT Scientific Publishing}}, title = {{{Optimizing the structure of nested dichotomies. A comparison of two heuristics}}}, year = {{2017}}, } @inproceedings{15399, author = {{Czech, M. and Hüllermeier, Eyke and Jacobs, M.C. and Wehrheim, Heike}}, booktitle = {{in Proceedings ESEC/FSE Workshops 2017 - 3rd ACM SIGSOFT, International Workshop on Software Analytics (SWAN 2017), Paderborn Germany}}, title = {{{Predicting rankings of software verification tools}}}, year = {{2017}}, } @inproceedings{15110, author = {{Couso, Ines and Dubois, D. and Hüllermeier, Eyke}}, booktitle = {{in Proceedings SUM 2017, 11th International Conference on Scalable Uncertainty Management, Granada, Spain}}, pages = {{3--16}}, publisher = {{Springer}}, title = {{{Maximum likelihood estimation and coarse data}}}, year = {{2017}}, } @inproceedings{10204, author = {{Ewerth, Ralph and Springstein, M. and Müller, E. and Balz, A. and Gehlhaar, J. and Naziyok, T. and Dembczynski, K. and Hüllermeier, Eyke}}, booktitle = {{Proc. IEEE Int. Conf. on Multimedia and Expo (ICME 2017)}}, pages = {{919--924}}, title = {{{Estimating relative depth in single images via rankboost}}}, year = {{2017}}, } @inproceedings{10205, author = {{Ahmadi Fahandar, Mohsen and Hüllermeier, Eyke and Couso, Ines}}, booktitle = {{Proc. 34th Int. Conf. on Machine Learning (ICML 2017)}}, pages = {{1078--1087}}, title = {{{Statistical Inference for Incomplete Ranking Data: The Case of Rank-Dependent Coarsening}}}, year = {{2017}}, } @inproceedings{10206, author = {{Mohr, Felix and Lettmann, Theodor and Hüllermeier, Eyke}}, booktitle = {{Proc. 40th Annual German Conference on Advances in Artificial Intelligence (KI 2017)}}, pages = {{193--206}}, title = {{{Planning with Independent Task Networks}}}, doi = {{10.1007/978-3-319-67190-1_15}}, year = {{2017}}, } @inproceedings{10207, author = {{Czech, M. and Hüllermeier, Eyke and Jakobs, M.-C. and Wehrheim, Heike}}, booktitle = {{Proc. 3rd ACM SIGSOFT Int. I Workshop on Software Analytics (SWAN@ESEC/SIGSOFT FSE 2017}}, pages = {{23--26}}, title = {{{Predicting rankings of software verification tools}}}, year = {{2017}}, } @inproceedings{10208, author = {{Couso, Ines and Dubois, D. and Hüllermeier, Eyke}}, booktitle = {{Proc. 11th Int. Conf. on Scalable Uncertainty Management (SUM 2017)}}, pages = {{3--16}}, title = {{{Maximum Likelihood Estimation and Coarse Data}}}, year = {{2017}}, } @inproceedings{10209, author = {{Ahmadi Fahandar, Mohsen and Hüllermeier, Eyke}}, booktitle = {{Proc. AAAI 2017, 32nd AAAI Conference on Artificial Intelligence}}, title = {{{Learning to Rank based on Analogical Reasoning}}}, year = {{2017}}, } @inproceedings{10212, author = {{Hoffmann, F. and Hüllermeier, Eyke and Mikut, R.}}, title = {{{(Hrsg.) Proceedings 27. Workshop Computational Intelligence, KIT Scientific Publishing, Karlsruhe, Germany 2017}}}, year = {{2017}}, }