@article{48777, abstract = {{AbstractExplainable artificial intelligence has mainly focused on static learning scenarios so far. We are interested in dynamic scenarios where data is sampled progressively, and learning is done in an incremental rather than a batch mode. We seek efficient incremental algorithms for computing feature importance (FI). Permutation feature importance (PFI) is a well-established model-agnostic measure to obtain global FI based on feature marginalization of absent features. We propose an efficient, model-agnostic algorithm called iPFI to estimate this measure incrementally and under dynamic modeling conditions including concept drift. We prove theoretical guarantees on the approximation quality in terms of expectation and variance. To validate our theoretical findings and the efficacy of our approaches in incremental scenarios dealing with streaming data rather than traditional batch settings, we conduct multiple experimental studies on benchmark data with and without concept drift.}}, author = {{Fumagalli, Fabian and Muschalik, Maximilian and Hüllermeier, Eyke and Hammer, Barbara}}, issn = {{0885-6125}}, journal = {{Machine Learning}}, keywords = {{Artificial Intelligence, Software}}, publisher = {{Springer Science and Business Media LLC}}, title = {{{Incremental permutation feature importance (iPFI): towards online explanations on data streams}}}, doi = {{10.1007/s10994-023-06385-y}}, year = {{2023}}, } @article{50262, abstract = {{AbstractExplainable artificial intelligence has mainly focused on static learning scenarios so far. We are interested in dynamic scenarios where data is sampled progressively, and learning is done in an incremental rather than a batch mode. We seek efficient incremental algorithms for computing feature importance (FI). Permutation feature importance (PFI) is a well-established model-agnostic measure to obtain global FI based on feature marginalization of absent features. We propose an efficient, model-agnostic algorithm called iPFI to estimate this measure incrementally and under dynamic modeling conditions including concept drift. We prove theoretical guarantees on the approximation quality in terms of expectation and variance. To validate our theoretical findings and the efficacy of our approaches in incremental scenarios dealing with streaming data rather than traditional batch settings, we conduct multiple experimental studies on benchmark data with and without concept drift.}}, author = {{Fumagalli, Fabian and Muschalik, Maximilian and Hüllermeier, Eyke and Hammer, Barbara}}, issn = {{0885-6125}}, journal = {{Machine Learning}}, keywords = {{Artificial Intelligence, Software}}, number = {{12}}, pages = {{4863--4903}}, publisher = {{Springer Science and Business Media LLC}}, title = {{{Incremental permutation feature importance (iPFI): towards online explanations on data streams}}}, doi = {{10.1007/s10994-023-06385-y}}, volume = {{112}}, year = {{2023}}, } @article{25035, abstract = {{AbstractThe efficiency of state-of-the-art algorithms for the dueling bandits problem is essentially due to a clever exploitation of (stochastic) transitivity properties of pairwise comparisons: If one arm is likely to beat a second one, which in turn is likely to beat a third one, then the first is also likely to beat the third one. By now, however, there is no way to test the validity of corresponding assumptions, although this would be a key prerequisite to guarantee the meaningfulness of the results produced by an algorithm. In this paper, we investigate the problem of testing different forms of stochastic transitivity in an online manner. We derive lower bounds on the expected sample complexity of any sequential hypothesis testing algorithm for various forms of stochastic transitivity, thereby providing additional motivation to focus on weak stochastic transitivity. To this end, we introduce an algorithmic framework for the dueling bandits problem, in which the statistical validity of weak stochastic transitivity can be tested, either actively or passively, based on a multiple binomial hypothesis test. Moreover, by exploiting a connection between weak stochastic transitivity and graph theory, we suggest an enhancement to further improve the efficiency of the testing algorithm. In the active setting, both variants achieve an expected sample complexity that is optimal up to a logarithmic factor.}}, author = {{Haddenhorst, Björn and Bengs, Viktor and Hüllermeier, Eyke}}, issn = {{0885-6125}}, journal = {{Machine Learning}}, pages = {{2063--2084}}, title = {{{On testing transitivity in online preference learning}}}, doi = {{10.1007/s10994-021-06026-2}}, year = {{2021}}, } @article{66216, abstract = {{AbstractThe predictive performance of a machine learning model highly depends on the corresponding hyper-parameter setting. Hence, hyper-parameter tuning is often indispensable. Normally such tuning requires the dedicated machine learning model to be trained and evaluated on centralized data to obtain a performance estimate. However, in a distributed machine learning scenario, it is not always possible to collect all the data from all nodes due to privacy concerns or storage limitations. Moreover, if data has to be transferred through low bandwidth connections it reduces the time available for tuning. Model-Based Optimization (MBO) is one state-of-the-art method for tuning hyper-parameters but the application on distributed machine learning models or federated learning lacks research. This work proposes a framework$$\textit{MODES}$$MODESthat allows to deploy MBO on resource-constrained distributed embedded systems. Each node trains an individual model based on its local data. The goal is to optimize the combined prediction accuracy. The presented framework offers two optimization modes: (1)$$\textit{MODES}$$MODES-B considers the whole ensemble as a single black box and optimizes the hyper-parameters of each individual model jointly, and (2)$$\textit{MODES}$$MODES-I considers all models as clones of the same black box which allows it to efficiently parallelize the optimization in a distributed setting. We evaluate$$\textit{MODES}$$MODESby conducting experiments on the optimization for the hyper-parameters of a random forest and a multi-layer perceptron. The experimental results demonstrate that, with an improvement in terms of mean accuracy ($$\textit{MODES}$$MODES-B), run-time efficiency ($$\textit{MODES}$$MODES-I), and statistical stability for both modes,$$\textit{MODES}$$MODESoutperforms the baseline, i.e., carry out tuning with MBO on each node individually with its local sub-data set.}}, author = {{Shi, Junjie and Bian, Jiang and Richter, Jakob and Chen, Kuan-Hsun and Rahnenführer, Jörg and Xiong, Haoyi and Chen, Jian-Jia}}, issn = {{0885-6125}}, journal = {{Machine Learning}}, number = {{6}}, pages = {{1527--1547}}, publisher = {{Springer Science and Business Media LLC}}, title = {{{MODES: model-based optimization on distributed embedded systems}}}, doi = {{10.1007/s10994-021-06014-6}}, volume = {{110}}, year = {{2021}}, } @article{3510, abstract = {{Automated machine learning (AutoML) seeks to automatically select, compose, and parametrize machine learning algorithms, so as to achieve optimal performance on a given task (dataset). Although current approaches to AutoML have already produced impressive results, the field is still far from mature, and new techniques are still being developed. In this paper, we present ML-Plan, a new approach to AutoML based on hierarchical planning. To highlight the potential of this approach, we compare ML-Plan to the state-of-the-art frameworks Auto-WEKA, auto-sklearn, and TPOT. In an extensive series of experiments, we show that ML-Plan is highly competitive and often outperforms existing approaches.}}, author = {{Mohr, Felix and Wever, Marcel Dominik and Hüllermeier, Eyke}}, issn = {{1573-0565}}, journal = {{Machine Learning}}, keywords = {{AutoML, Hierarchical Planning, HTN planning, ML-Plan}}, location = {{Dublin, Ireland}}, pages = {{1495--1515}}, publisher = {{Springer}}, title = {{{ML-Plan: Automated Machine Learning via Hierarchical Planning}}}, doi = {{10.1007/s10994-018-5735-z}}, year = {{2018}}, }