@article{21004,
abstract = {Automated machine learning (AutoML) supports the algorithmic construction and data-specific customization of machine learning pipelines, including the selection, combination, and parametrization of machine learning algorithms as main constituents. Generally speaking, AutoML approaches comprise two major components: a search space model and an optimizer for traversing the space. Recent approaches have shown impressive results in the realm of supervised learning, most notably (single-label) classification (SLC). Moreover, first attempts at extending these approaches towards multi-label classification (MLC) have been made. While the space of candidate pipelines is already huge in SLC, the complexity of the search space is raised to an even higher power in MLC. One may wonder, therefore, whether and to what extent optimizers established for SLC can scale to this increased complexity, and how they compare to each other. This paper makes the following contributions: First, we survey existing approaches to AutoML for MLC. Second, we augment these approaches with optimizers not previously tried for MLC. Third, we propose a benchmarking framework that supports a fair and systematic comparison. Fourth, we conduct an extensive experimental study, evaluating the methods on a suite of MLC problems. We find a grammar-based best-first search to compare favorably to other optimizers.},
author = {Wever, Marcel Dominik and Tornede, Alexander and Mohr, Felix and Hüllermeier, Eyke},
issn = {0162-8828},
journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
keyword = {Automated Machine Learning, Multi Label Classification, Hierarchical Planning, Bayesian Optimization},
pages = {1--1},
title = {{AutoML for Multi-Label Classification: Overview and Empirical Evaluation}},
doi = {10.1109/tpami.2021.3051276},
year = {2021},
}
@article{21092,
abstract = {Automated Machine Learning (AutoML) seeks to automatically find so-called machine learning pipelines that maximize the prediction performance when being used to train a model on a given dataset. One of the main and yet open challenges in AutoML is an effective use of computational resources: An AutoML process involves the evaluation of many candidate pipelines, which are costly but often ineffective because they are canceled due to a timeout.
In this paper, we present an approach to predict the runtime of two-step machine learning pipelines with up to one pre-processor, which can be used to anticipate whether or not a pipeline will time out. Separate runtime models are trained offline for each algorithm that may be used in a pipeline, and an overall prediction is derived from these models. We empirically show that the approach increases successful evaluations made by an AutoML tool while preserving or even improving on the previously best solutions.},
author = {Mohr, Felix and Wever, Marcel Dominik and Tornede, Alexander and Hüllermeier, Eyke},
journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
publisher = {IEEE},
title = {{Predicting Machine Learning Pipeline Runtimes in the Context of Automated Machine Learning}},
year = {2021},
}
@inproceedings{21198,
author = {Hanselle, Jonas Manuel and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke},
location = {Delhi, India},
title = {{Algorithm Selection as Superset Learning: Constructing Algorithm Selectors from Imprecise Performance Data}},
year = {2021},
}
@article{21535,
author = {Bengs, Viktor and Busa-Fekete, Róbert and El Mesaoudi-Paul, Adil and Hüllermeier, Eyke},
journal = {Journal of Machine Learning Research},
number = {7},
pages = {1--108},
title = {{Preference-based Online Learning with Dueling Bandits: A Survey}},
volume = {22},
year = {2021},
}
@unpublished{21600,
abstract = {Many problems in science and engineering require the efficient numerical
approximation of integrals, a particularly important application being the
numerical solution of initial value problems for differential equations. For
complex systems, an equidistant discretization is often inadvisable, as it
either results in prohibitively large errors or computational effort. To this
end, adaptive schemes have been developed that rely on error estimators based
on Taylor series expansions. While these estimators a) rely on strong
smoothness assumptions and b) may still result in erroneous steps for complex
systems (and thus require step rejection mechanisms), we here propose a
data-driven time stepping scheme based on machine learning, and more
specifically on reinforcement learning (RL) and meta-learning. First, one or
several (in the case of non-smooth or hybrid systems) base learners are trained
using RL. Then, a meta-learner is trained which (depending on the system state)
selects the base learner that appears to be optimal for the current situation.
Several examples including both smooth and non-smooth problems demonstrate the
superior performance of our approach over state-of-the-art numerical schemes.
The code is available under https://github.com/lueckem/quadrature-ML.},
author = {Dellnitz, Michael and Hüllermeier, Eyke and Lücke, Marvin and Ober-Blöbaum, Sina and Offen, Christian and Peitz, Sebastian and Pfannschmidt, Karlson},
booktitle = {arXiv:2104.03562},
title = {{Efficient time stepping for numerical integration using reinforcement learning}},
year = {2021},
}
@inproceedings{19953,
abstract = {Current GNN architectures use a vertex neighborhood aggregation scheme, which limits their discriminative power to that of the 1-dimensional Weisfeiler-Lehman (WL) graph isomorphism test. Here, we propose a novel graph convolution operator that is based on the 2-dimensional WL test. We formally show that the resulting 2-WL-GNN architecture is more discriminative than existing GNN approaches. This theoretical result is complemented by experimental studies using synthetic and real data. On multiple common graph classification benchmarks, we demonstrate that the proposed model is competitive with state-of-the-art graph kernels and GNNs.},
author = {Damke, Clemens and Melnikov, Vitaly and Hüllermeier, Eyke},
booktitle = {Proceedings of the 12th Asian Conference on Machine Learning (ACML 2020)},
editor = {Jialin Pan, Sinno and Sugiyama, Masashi},
keyword = {graph neural networks, Weisfeiler-Lehman test, cycle detection},
location = {Bangkok, Thailand},
pages = {49--64},
publisher = {PMLR},
title = {{A Novel Higher-order Weisfeiler-Lehman Graph Convolution}},
volume = {129},
year = {2020},
}
@inproceedings{17407,
author = {Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke},
booktitle = {Discovery Science},
title = {{Extreme Algorithm Selection with Dyadic Feature Representation}},
year = {2020},
}
@inproceedings{17408,
author = {Hanselle, Jonas Manuel and Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke},
booktitle = {KI 2020: Advances in Artificial Intelligence},
title = {{Hybrid Ranking and Regression for Algorithm Selection}},
year = {2020},
}
@unpublished{17605,
abstract = {Syntactic annotation of corpora in the form of part-of-speech (POS) tags is a key requirement for both linguistic research and subsequent automated natural language processing (NLP) tasks. This problem is commonly tackled using machine learning methods, i.e., by training a POS tagger on a sufficiently large corpus of labeled data.
While the problem of POS tagging can essentially be considered as solved for modern languages, historical corpora turn out to be much more difficult, especially due to the lack of native speakers and sparsity of training data. Moreover, most texts have no sentences as we know them today, nor a common orthography.
These irregularities render the task of automated POS tagging more difficult and error-prone. Under these circumstances, instead of forcing the POS tagger to predict and commit to a single tag, it should be enabled to express its uncertainty. In this paper, we consider POS tagging within the framework of set-valued prediction, which allows the POS tagger to express its uncertainty via predicting a set of candidate POS tags instead of guessing a single one. The goal is to guarantee a high confidence that the correct POS tag is included while keeping the number of candidates small.
In our experimental study, we find that extending state-of-the-art POS taggers to set-valued prediction yields more precise and robust taggings, especially for unknown words, i.e., words not occurring in the training data.},
author = {Heid, Stefan Helmut and Wever, Marcel Dominik and Hüllermeier, Eyke},
booktitle = {Journal of Data Mining and Digital Humanities},
publisher = {episciences},
title = {{Reliable Part-of-Speech Tagging of Historical Corpora through Set-Valued Prediction}},
year = {2020},
}
@inbook{18014,
author = {El Mesaoudi-Paul, Adil and Weiß, Dimitri and Bengs, Viktor and Hüllermeier, Eyke and Tierney, Kevin},
booktitle = {Learning and Intelligent Optimization. LION 2020.},
isbn = {9783030535513},
issn = {0302-9743},
pages = {216 -- 232},
publisher = {Springer},
title = {{Pool-Based Realtime Algorithm Configuration: A Preselection Bandit Approach}},
doi = {10.1007/978-3-030-53552-0_22},
volume = {12096},
year = {2020},
}
@unpublished{21536,
abstract = {We consider a resource-aware variant of the classical multi-armed bandit
problem: In each round, the learner selects an arm and determines a resource
limit. It then observes a corresponding (random) reward, provided the (random)
amount of consumed resources remains below the limit. Otherwise, the
observation is censored, i.e., no reward is obtained. For this problem setting,
we introduce a measure of regret, which incorporates the actual amount of
allocated resources of each learning round as well as the optimality of
realizable rewards. Thus, to minimize regret, the learner needs to set a
resource limit and choose an arm in such a way that the chance to realize a
high reward within the predefined resource limit is high, while the resource
limit itself should be kept as low as possible. We derive the theoretical lower
bound on the cumulative regret and propose a learning algorithm having a regret
upper bound that matches the lower bound. In a simulation study, we show that
our learning algorithm outperforms straightforward extensions of standard
multi-armed bandit algorithms.},
author = {Bengs, Viktor and Hüllermeier, Eyke},
booktitle = {arXiv:2011.00813},
title = {{Multi-Armed Bandits with Censored Consumption of Resources}},
year = {2020},
}
@inproceedings{15629,
abstract = {In multi-label classification (MLC), each instance is associated with a set of class labels, in contrast to standard classification where an instance is assigned a single label. Binary relevance (BR) learning, which reduces a multi-label to a set of binary classification problems, one per label, is arguably the most straight-forward approach to MLC. In spite of its simplicity, BR proved to be competitive to more sophisticated MLC methods, and still achieves state-of-the-art performance for many loss functions. Somewhat surprisingly, the optimal choice of the base learner for tackling the binary classification problems has received very little attention so far. Taking advantage of the label independence assumption inherent to BR, we propose a label-wise base learner selection method optimizing label-wise macro averaged performance measures. In an extensive experimental evaluation, we find that or approach, called LiBRe, can significantly improve generalization performance.},
author = {Wever, Marcel Dominik and Tornede, Alexander and Mohr, Felix and Hüllermeier, Eyke},
location = {Konstanz, Germany},
publisher = {Springer},
title = {{LiBRe: Label-Wise Selection of Base Learners in Binary Relevance for Multi-Label Classification}},
year = {2020},
}
@inproceedings{20306,
author = {Tornede, Alexander and Wever, Marcel Dominik and Hüllermeier, Eyke},
booktitle = {Workshop MetaLearn 2020 @ NeurIPS 2020},
location = {Online},
title = {{Towards Meta-Algorithm Selection}},
year = {2020},
}
@inproceedings{17424,
author = {Tornede, Tanja and Tornede, Alexander and Wever, Marcel Dominik and Mohr, Felix and Hüllermeier, Eyke},
booktitle = {Proceedings of the ECMLPKDD 2020},
title = {{AutoML for Predictive Maintenance: One Tool to RUL Them All}},
doi = {10.1007/978-3-030-66770-2_8},
year = {2020},
}
@inbook{19521,
author = {Pfannschmidt, Karlson and Hüllermeier, Eyke},
booktitle = {Lecture Notes in Computer Science},
isbn = {9783030582845},
issn = {0302-9743},
title = {{Learning Choice Functions via Pareto-Embeddings}},
doi = {10.1007/978-3-030-58285-2_30},
year = {2020},
}
@article{16725,
author = {Richter, Cedric and Hüllermeier, Eyke and Jakobs, Marie-Christine and Wehrheim, Heike},
journal = {Journal of Automated Software Engineering},
publisher = {Springer},
title = {{Algorithm Selection for Software Validation Based on Graph Kernels}},
year = {2020},
}
@inproceedings{18276,
abstract = {Algorithm selection (AS) deals with the automatic selection of an algorithm
from a fixed set of candidate algorithms most suitable for a specific instance
of an algorithmic problem class, where "suitability" often refers to an
algorithm's runtime. Due to possibly extremely long runtimes of candidate
algorithms, training data for algorithm selection models is usually generated
under time constraints in the sense that not all algorithms are run to
completion on all instances. Thus, training data usually comprises censored
information, as the true runtime of algorithms timed out remains unknown.
However, many standard AS approaches are not able to handle such information in
a proper way. On the other side, survival analysis (SA) naturally supports
censored data and offers appropriate ways to use such data for learning
distributional models of algorithm runtime, as we demonstrate in this work. We
leverage such models as a basis of a sophisticated decision-theoretic approach
to algorithm selection, which we dub Run2Survive. Moreover, taking advantage of
a framework of this kind, we advocate a risk-averse approach to algorithm
selection, in which the avoidance of a timeout is given high priority. In an
extensive experimental study with the standard benchmark ASlib, our approach is
shown to be highly competitive and in many cases even superior to
state-of-the-art AS approaches.},
author = {Tornede, Alexander and Wever, Marcel Dominik and Werner, Stefan and Mohr, Felix and Hüllermeier, Eyke},
booktitle = {ACML 2020},
location = {Bangkok, Thailand},
title = {{Run2Survive: A Decision-theoretic Approach to Algorithm Selection based on Survival Analysis}},
year = {2020},
}
@unpublished{18017,
abstract = {We consider an extension of the contextual multi-armed bandit problem, in
which, instead of selecting a single alternative (arm), a learner is supposed
to make a preselection in the form of a subset of alternatives. More
specifically, in each iteration, the learner is presented a set of arms and a
context, both described in terms of feature vectors. The task of the learner is
to preselect $k$ of these arms, among which a final choice is made in a second
step. In our setup, we assume that each arm has a latent (context-dependent)
utility, and that feedback on a preselection is produced according to a
Plackett-Luce model. We propose the CPPL algorithm, which is inspired by the
well-known UCB algorithm, and evaluate this algorithm on synthetic and real
data. In particular, we consider an online algorithm selection scenario, which
served as a main motivation of our problem setting. Here, an instance (which
defines the context) from a certain problem class (such as SAT) can be solved
by different algorithms (the arms), but only $k$ of these algorithms can
actually be run.},
author = {El Mesaoudi-Paul, Adil and Bengs, Viktor and Hüllermeier, Eyke},
booktitle = {arXiv:2002.04275},
title = {{Online Preselection with Context Information under the Plackett-Luce Model}},
year = {2020},
}
@inproceedings{21534,
author = {Bengs, Viktor and Hüllermeier, Eyke},
booktitle = {International Conference on Machine Learning},
pages = {778--787},
title = {{Preselection Bandits}},
year = {2020},
}
@article{14028,
author = {Bengs, Viktor and Holzmann, Hajo},
issn = {1935-7524},
journal = {Electronic Journal of Statistics},
pages = {1523--1579},
title = {{Adaptive confidence sets for kink estimation}},
doi = {10.1214/19-ejs1555},
year = {2019},
}