mlr3measures/0000755000176200001440000000000015170431602012673 5ustar liggesusersmlr3measures/tests/0000755000176200001440000000000014472156310014041 5ustar liggesusersmlr3measures/tests/testthat/0000755000176200001440000000000015170431602015675 5ustar liggesusersmlr3measures/tests/testthat/test_classif_logloss.R0000644000176200001440000000046415111057476022261 0ustar liggesuserstest_that("logloss obs_loss works", { truth = factor(rep(c("a", "b"), each = 5), levels = c("a", "b")) prob = matrix(runif(length(truth) * 2), ncol = 2) prob = t(apply(prob, 1, function(x) x / sum(x))) colnames(prob) = levels(truth) expect_numeric(obs_logloss(truth, prob), len = length(truth)) }) mlr3measures/tests/testthat/test_similarity.R0000644000176200001440000000250414472156310021252 0ustar liggesusersrun_all_measures = function(sets, p) { tol = sqrt(.Machine$double.eps) for (m in as.list(measures)) { if (m$type != "similarity") { next } f = match.fun(m$id) perf = f(sets, p = p) expect_number(perf, na.ok = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) } } test_that("trigger all", { sets = lapply(1:3, function(i) sample(letters[1:5], 2)) run_all_measures(sets, p = 5) }) test_that("no similarity", { p = 4L sets = list( 1:2, 3:4 ) expect_equal(jaccard(sets), measures$jaccard$lower) expect_equal(phi(sets, p = 4), measures$phi$lower) }) test_that("perfect similarity", { p = 2L sets = list( 1:2, 1:2 ) expect_equal(jaccard(sets), measures$jaccard$upper) expect_equal(phi(sets, p = 4), measures$phi$upper) }) test_that("jaccard", { sets = list( integer(), integer(), 1:2 ) expect_identical(jaccard(sets), NaN) sets = list( integer(), 1:2 ) expect_equal(jaccard(sets), 0) }) test_that("phi", { sets = list( integer(), 1:2 ) expect_identical(phi(sets, p = 2), NaN) sets = list( 1L, 1:2 ) expect_identical(phi(sets, p = 2), NaN) sets = list( 1L, 2L ) expect_equal(phi(sets, p = 2), -1) sets = list( 1:3L, 2L ) expect_error(phi(sets, p = 2), "exceeds") }) mlr3measures/tests/testthat/helper.R0000644000176200001440000000022515170127602017300 0ustar liggesuserslibrary(checkmate) ssample = function(lvls, N) { x = c(lvls, sample(lvls, N - length(lvls), replace = TRUE)) factor(sample(x), levels = lvls) } mlr3measures/tests/testthat/test_binary.R0000644000176200001440000002030015170376754020356 0ustar liggesusersrun_all_measures = function(truth, response, prob, positive, na_allowed = FALSE) { conf = cm(truth, response, positive = positive) na_value = if (na_allowed) 123456789 else NaN tol = sqrt(.Machine$double.eps) sample_weights = runif(length(truth)) for (m in as.list(measures)) { if (m$type != "binary") { next } f = match.fun(m$id) f_cm = get0(sprintf("%s_cm", m$id)) perf = f(truth = truth, response = response, prob = prob, positive = positive, na_value = na_value) if (!(na_allowed && identical(perf, na_value))) { expect_number(perf, na.ok = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) } if (!is.null(f_cm)) { # nolint next expect_identical(perf, f_cm(conf, na_value = na_value), label = m$id) } if ("sample_weights" %in% names(formals(f))) { perf = f( truth = truth, response = response, prob = prob, positive = positive, na_value = na_value, sample_weights = sample_weights ) expect_number(perf, na.ok = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) } } } test_that("trigger all", { N = 30L truth = ssample(letters[1:2], N) response = ssample(letters[1:2], N) prob = runif(N) positive = sample(letters[1:2], 1) run_all_measures(truth, response, prob, positive) }) test_that("integer overflow", { N = 500000 truth = ssample(c("a", "b"), N) response = truth prob = runif(N) positive = sample(letters[1:2], 1) run_all_measures(truth, response, prob, positive, na_allowed = TRUE) response = ssample(c("a", "b"), N) run_all_measures(truth, response, prob, positive) response = factor(ifelse(truth == "a", "b", "a"), levels = levels(truth)) run_all_measures(truth, response, prob, positive, na_allowed = TRUE) }) test_that("tests from Metrics", { as_fac = function(...) factor(ifelse(c(...) == 0, "b", "a"), levels = c("a", "b")) as_prob = function(...) { p = c(...) p = cbind(p, 1 - p) colnames(p) = c("a", "b") p } expect_equal(auc(as_fac(1, 0, 1, 1), c(.32, .52, .26, .86), "a"), 1 / 3) expect_equal(auc(as_fac(1, 0, 1, 0, 1), c(.9, .1, .8, .1, .7), "a"), 1) expect_equal(auc(as_fac(0, 1, 1, 0), c(.2, .1, .3, .4), "a"), 1 / 4) expect_equal(auc(as_fac(1, 1, 1, 1, 0, 0, 0, 0, 0, 0), 0 * (1:10), "a"), 0.5) # expect_equal(ll(1,1), 0) # expect_equal(ll(1,0), Inf) # expect_equal(ll(0,1), Inf) # expect_equal(ll(1,0.5), -log(0.5)) expect_equal(ppv(as_fac(1, 1, 0, 0), as_fac(1, 1, 0, 0), "a"), 1) expect_equal(ppv(as_fac(0, 0, 1, 1), as_fac(1, 1, 0, 0), "a"), 0) expect_equal(ppv(as_fac(1, 1, 0, 0), as_fac(1, 1, 1, 1), "a"), 1 / 2) expect_equal(tpr(as_fac(1, 1, 0, 0), as_fac(1, 1, 0, 0), "a"), 1) expect_equal(tpr(as_fac(0, 0, 1, 1), as_fac(1, 1, 0, 0), "a"), 0) expect_equal(tpr(as_fac(1, 1, 1, 1), as_fac(1, 0, 0, 1), "a"), 1 / 2) expect_equal(fbeta(as_fac(1, 1, 0, 0), as_fac(1, 1, 0, 0), "a"), 1) expect_equal(fbeta(as_fac(0, 0, 1, 1), as_fac(1, 1, 1, 0), "a"), 2 / 5) expect_equal(fbeta(as_fac(1, 1, 1, 1), as_fac(1, 0, 0, 1), "a"), 2 / 3) expect_equal(fbeta(as_fac(1, 1, 0, 0), as_fac(1, 1, 1, 1), "a", beta = 0), 1 / 2) }) test_that("confusion measures", { truth = factor(rep(c("a", "b"), each = 5), levels = c("a", "b")) response = factor(c("a", "a", "a", "b", "b", "b", "b", "b", "b", "a"), levels = c("a", "b")) # table(response, truth) TP = tp(response, truth, positive = "a") TN = tn(response, truth, positive = "a") FP = fp(response, truth, positive = "a") FN = fn(response, truth, positive = "a") expect_identical(TP, 3L) expect_identical(TN, 4L) expect_identical(FP, 2L) expect_identical(FN, 1L) expect_equal(dor(response, truth, positive = "a"), (TP / FP) / (FN / TN)) expect_equal(fdr(response, truth, positive = "a"), FP / (TP + FP)) expect_equal(fnr(response, truth, positive = "a"), FN / (TP + FN)) expect_equal(fomr(response, truth, positive = "a"), FN / (FN + TN)) expect_equal(fpr(response, truth, positive = "a"), FP / (FP + TN)) expect_equal( mcc(response, truth, positive = "a"), (TP * TN - FP * FN) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)) ) expect_equal(npv(response, truth, positive = "a"), TN / (FN + TN)) expect_equal(ppv(response, truth, positive = "a"), TP / (TP + FP)) expect_equal(precision(response, truth, positive = "a"), TP / (TP + FP)) expect_equal(recall(response, truth, positive = "a"), TP / (TP + FN)) expect_equal(sensitivity(response, truth, positive = "a"), TP / (TP + FN)) expect_equal(specificity(response, truth, positive = "a"), TN / (FP + TN)) expect_equal(tnr(response, truth, positive = "a"), TN / (FP + TN)) expect_equal(tpr(response, truth, positive = "a"), TP / (TP + FN)) }) test_that("weighted confusion measures", { truth = factor(c("a", "a", "b", "b"), levels = c("a", "b")) response = factor(c("a", "b", "a", "b"), levels = c("a", "b")) # Unweighted: TP=1, TN=1, FP=1, FN=1 # Weighted (2,1,1,2): TP=2, TN=2, FP=1, FN=1 w = c(2, 1, 1, 2) # Weighted counts TP = tp(truth, response, positive = "a", sample_weights = w) TN = tn(truth, response, positive = "a", sample_weights = w) FP = fp(truth, response, positive = "a", sample_weights = w) FN = fn(truth, response, positive = "a", sample_weights = w) expect_equal(TP, 2) expect_equal(TN, 2) expect_equal(FP, 1) expect_equal(FN, 1) # Derived measures with weights expect_equal(tpr(truth, response, positive = "a", sample_weights = w), TP / (TP + FN)) expect_equal(fpr(truth, response, positive = "a", sample_weights = w), FP / (FP + TN)) expect_equal(tnr(truth, response, positive = "a", sample_weights = w), TN / (FP + TN)) expect_equal(fnr(truth, response, positive = "a", sample_weights = w), FN / (TP + FN)) expect_equal(ppv(truth, response, positive = "a", sample_weights = w), TP / (TP + FP)) expect_equal(npv(truth, response, positive = "a", sample_weights = w), TN / (FN + TN)) expect_equal(fdr(truth, response, positive = "a", sample_weights = w), FP / (TP + FP)) expect_equal(fomr(truth, response, positive = "a", sample_weights = w), FN / (FN + TN)) expect_equal(dor(truth, response, positive = "a", sample_weights = w), (TP / FP) / (FN / TN)) expect_equal( mcc(truth, response, positive = "a", sample_weights = w), (TP * TN - FP * FN) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)) ) # Composite measures expect_equal(fbeta(truth, response, positive = "a", sample_weights = w), 2 * TP / (2 * TP + FP + FN)) expect_equal(gmean(truth, response, positive = "a", sample_weights = w), sqrt((TP / (TP + FN)) * (TN / (FP + TN)))) expect_equal(gpr(truth, response, positive = "a", sample_weights = w), sqrt((TP / (TP + FP)) * (TP / (TP + FN)))) # Uniform weights should equal unweighted w_uniform = c(1, 1, 1, 1) expect_equal(tpr(truth, response, positive = "a", sample_weights = w_uniform), tpr(truth, response, positive = "a")) expect_equal( fbeta(truth, response, positive = "a", sample_weights = w_uniform), fbeta(truth, response, positive = "a") ) }) test_that("weighted auc", { truth = factor(c("a", "a", "b", "b"), levels = c("a", "b")) prob = c(0.9, 0.6, 0.7, 0.2) # uniform weights equal unweighted expect_equal( auc(truth, prob, "a", sample_weights = rep(1, 4)), auc(truth, prob, "a") ) # duplicating observations via integer weights equals repeating them w = c(2, 1, 1, 2) truth_rep = truth[rep(seq_along(truth), w)] prob_rep = prob[rep(seq_along(prob), w)] expect_equal( auc(truth, prob, "a", sample_weights = w), auc(truth_rep, prob_rep, "a") ) # degenerate weights return na_value expect_equal(auc(truth, prob, "a", sample_weights = c(0, 0, 1, 1), na_value = NA_real_), NA_real_) expect_equal(auc(truth, prob, "a", sample_weights = c(1, 1, 0, 0), na_value = NA_real_), NA_real_) }) test_that("bbrier", { N = 30L truth = ssample(letters[1:2], N) prob = as.numeric(truth == "a") expect_equal(bbrier(truth, prob, positive = "a"), 0) expect_equal(bbrier(truth, prob, positive = "b"), 1) prob = runif(N) pm = cbind(prob, 1 - prob) colnames(pm) = c("a", "b") expect_equal(2 * bbrier(truth, prob, "a"), mbrier(truth, pm)) colnames(pm) = c("b", "a") expect_equal(2 * bbrier(truth, prob, "b"), mbrier(truth, pm)) }) mlr3measures/tests/testthat/test_regr.R0000644000176200001440000000501215056012454020017 0ustar liggesuserstest_that("trigger all", { N = 10L truth = 1 + runif(N) response = 1 + runif(N) tol = sqrt(.Machine$double.eps) sample_weights = runif(length(truth)) for (m in as.list(measures)) { if (m$type != "regr") { next } f = match.fun(m$id) perf = f(truth = truth, response = response) if (m$aggregated) { expect_number(perf, na.ok = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) if ("sample_weights" %in% names(formals(f))) { perf = f(truth = truth, response = response, sample_weights = sample_weights) expect_number(perf, na.ok = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) } } else { expect_numeric(perf, len = N, any.missing = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) } } }) test_that("tests from Metrics", { expect_equal(bias(1, 1), 0) expect_equal(bias(c(-1, -100, 17.5), c(0, 0, 0)), mean(c(1, 100, -17.5))) expect_equal(pbias(c(1, 2, 3), c(1, 3, 2)), mean(c(0, 1 / 2, -1 / 3))) expect_equal(pbias(c(1, 2, 0), c(1, 2, 1)), NaN) expect_equal(pbias(0, 0), NaN) expect_equal(pbias(c(-1.1, 1.1), c(-1, 1)), 0) expect_equal(se(3.4, 4.4), 1) expect_equal(se(9:11, 11:9), c(4, 0, 4)) expect_equal(sse(c(1, 3, 2), c(2, 3, 4)), 5) expect_equal(mse(1:4, c(2, 3, 4, 4)), 3 / 4) expect_equal(rmse(1:4, c(1, 2, 3, 5)), sqrt(1 / 4)) expect_equal(rmse(1:4, 1:4), 0) expect_equal(ae(3.4, 4.4), 1) expect_equal(ae(9:11, 11:9), c(2, 0, 2)) expect_equal(mae(1:4, c(1, 2, 3, 5)), 0.25) expect_equal(medae(1:4, c(1, 2, 4, 50)), 0.5) expect_equal(ape(0:3, 1:4), c(Inf, 1, 1 / 2, 1 / 3)) expect_equal(ape(0:2, c(0, 0, 0)), c(NaN, 1, 1)) expect_equal(ape(c(-1.1, 1.1), c(-1, 1)), c(1 / 11, 1 / 11)) expect_equal(mape(1:3, 2:4), mean(c(1, 1 / 2, 1 / 3))) expect_equal(mape(c(-1.1, 1.1), c(-1, 1)), 1 / 11) expect_equal(smape(0, 0), NaN) expect_equal(smape(1, -1), 2) expect_equal(smape(1, 0), 2) expect_equal(smape(c(1, 2, 3), c(2, 5, 4)), smape(c(2, 5, 4), c(1, 2, 3))) expect_equal(sle(c(0, 1, 3.4), c(1, 0, 3.4)), c(log(2), log(2), 0)^2) expect_equal(sle(exp(2) - 1, exp(1) - 1), 1) expect_equal(msle(c(1, 2, exp(1) - 1), c(1, 2, exp(2) - 1)), 1 / 3) expect_equal(rmsle(c(exp(5) - 1), c(exp(1) - 1)), 4) expect_equal(pinball(1:3, 1:3), 0) expect_equal(pinball(1:3, c(0, 2, 3)), 1 / 6) expect_equal(pinball(1:3, c(1, 2, 4)), 1 / 6) expect_equal(pinball(1:3, c(1, 2, 4), alpha = 0.1), 0.3) expect_equal(pinball(1:3, c(1, 2, 4), alpha = 0.4), 0.2) }) mlr3measures/tests/testthat/test_binary_bbrier.R0000644000176200001440000000032715111057476021702 0ustar liggesuserstest_that("bbrier obs_loss works", { truth = factor(rep(c("a", "b"), each = 5), levels = c("a", "b")) prob = runif(length(truth)) expect_numeric(se_binary(truth, prob, positive = "a"), len = length(truth)) }) mlr3measures/tests/testthat/test_classif.R0000644000176200001440000001730615170376754020532 0ustar liggesusersrun_all_measures = function(truth, response, prob) { tol = sqrt(.Machine$double.eps) sample_weights = runif(length(truth)) for (m in as.list(measures)) { if (m$type != "classif") { next } f = match.fun(m$id) perf = f(truth = truth, response = response, prob = prob) if (m$aggregated) { expect_number(perf, na.ok = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) if ("sample_weights" %in% names(formals(f))) { perf = f(truth = truth, response = response, prob = prob, sample_weights = sample_weights) expect_number(perf, na.ok = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) } } else { expect_numeric(perf, any.missing = FALSE, lower = m$lower - tol, upper = m$upper + tol, label = m$id) } } } test_that("trigger all", { k = 3 n = 10 truth = ssample(letters[1:k], n) response = ssample(letters[1:k], n) prob = matrix(runif(n * k, min = 1e-8, max = 1 - 1e-8), nrow = n) prob = t(apply(prob, 1, function(x) x / sum(x))) colnames(prob) = letters[1:k] run_all_measures(truth, response, prob) }) test_that("integer overflow", { N = 500000 truth = ssample(c("a", "b"), N) response = truth prob = matrix(runif(N * 2), ncol = 2) prob = t(apply(prob, 1, function(x) x / sum(x))) colnames(prob) = levels(truth) run_all_measures(truth, response, prob) response = ssample(c("a", "b"), N) run_all_measures(truth, response, prob) response = factor(ifelse(truth == "a", "b", "a"), levels = levels(truth)) run_all_measures(truth, response, prob) }) test_that("tests from Metrics", { as_fac = function(...) factor(ifelse(c(...) == 0, "b", "a"), levels = c("a", "b")) as_prob = function(...) { p = c(...) p = cbind(p, 1 - p) colnames(p) = c("a", "b") p } expect_equal(ce(as_fac(1, 1, 1, 0, 0, 0), as_fac(1, 1, 1, 0, 0, 0)), 0.0) expect_equal(ce(as_fac(1, 1, 1, 0, 0, 0), as_fac(1, 1, 1, 1, 0, 0)), 1 / 6) expect_equal(ce(factor(c(1, 2, 3, 4), levels = 1:4), factor(c(1, 2, 3, 3), levels = 1:4)), 1 / 4) lvls = c("cat", "dog", "bird", "fish") expect_equal( ce(factor(c("cat", "dog", "bird"), levels = lvls), factor(c("cat", "dog", "fish"), levels = lvls)), 1 / 3 ) expect_equal(logloss(as_fac(1, 1, 0, 0), as_prob(1, 1, 0, 0)), 0) expect_number(logloss(as_fac(1, 1, 0, 0), as_prob(0, 0, 1, 1)), lower = 10, upper = 50) expect_equal(logloss(as_fac(1, 1, 1, 0, 0, 0), as_prob(.5, .1, .01, .9, .75, .001)), 1.881797068998267) expect_equal(mcc(factor(1:4, levels = 1:4), factor(1:4, levels = 1:4)), 1) expect_equal(mcc(factor(1:4, levels = 1:4), factor(4:1, levels = 1:4)), -1 / 3) expect_equal( mcc(factor(c("cat", "dog", "bird"), levels = lvls), factor(c("cat", "dog", "fish"), levels = lvls)), 2 / 3 ) # rater.a <- c(1, 2, 1) # rater.b <- c(1, 2, 2) # kappa <- ScoreQuadraticWeightedKappa(rater.a, rater.b) # expect_equal(kappa, 0.4) # rater.a <- c(1, 2, 3, 1, 2, 3) # rater.b <- c(1, 2, 3, 1, 3, 2) # kappa <- ScoreQuadraticWeightedKappa(rater.a, rater.b) # expect_equal(kappa, 0.75) # rater.a <- c(1, 2, 3) # rater.b <- c(1, 2, 3) # kappa <- ScoreQuadraticWeightedKappa(rater.a, rater.b) # expect_equal(kappa, 1.0) # rater.a <- c(1, 3, 5) # rater.b <- c(2, 4, 6) # kappa <- ScoreQuadraticWeightedKappa(rater.a, rater.b) # expect_equal(kappa, 0.8421052631578947) # rater.a <- c(1, 3, 3, 5) # rater.b <- c(2, 4, 5, 6) # kappa <- ScoreQuadraticWeightedKappa(rater.a, rater.b, 1, 6) # expect_equal(kappa, 0.6956521739130435) # # # kappa <- MeanQuadraticWeightedKappa( c(1, 1) ) # expect_equal(kappa, 0.999) # kappa <- MeanQuadraticWeightedKappa( c(1, -1) ) # expect_equal(kappa, 0.0) # kappa <- MeanQuadraticWeightedKappa( c(.5, .8), c(1.0, .5) ) # expect_equal(kappa, 0.624536446425734) }) test_that("bacc", { truth = factor(c("a", "a", "b", "b"), levels = c("a", "b")) response = factor(c("a", "a", "b", "a"), levels = c("a", "b")) expect_equal(bacc(truth, response), 0.75) expect_equal(bacc(truth, response, sample_weights = c(0.25, 0.25, 0.25, 0.25)), 0.75) expect_equal(bacc(truth, response, sample_weights = c(0.25, 0.25, 0.25, 1)), 0.6) truth = factor(c("a", "a", "a", "a", "a", "b"), levels = c("a", "b")) response = factor(c("a", "a", "a", "a", "b", "b"), levels = c("a", "b")) expect_equal(bacc(truth, response), 0.9) expect_equal(bacc(truth, response, sample_weights = c(0, 0, 0, 0, 0, 1)), 1) expect_equal(bacc(truth, response, sample_weights = c(0, 0, 0, 0, 0.5, 0.5)), 0.5) truth = factor(c("c", "a", "a", "a", "a", "b"), levels = c("a", "b", "c")) response = factor(c("c", "a", "a", "a", "b", "b"), levels = c("a", "b", "c")) expect_equal(round(bacc(truth, response), 3), 0.917) }) # test_that("ber", { # truth = factor(c("a", "a", "b", "b", "c", "c"), levels = c("a", "b", "c")) # response = factor(c("a", "a", "b", "b", "c", "c"), levels = c("a", "b", "c")) # expect_equal(ber(truth, response), 0) # response = factor(c("a", "b", "b", "c", "c", "a"), levels = c("a", "b", "c")) # expect_equal(ber(truth, response), 0.5) # response = factor(rep("a", 6), levels = c("a", "b", "c")) # expect_equal(round(ber(truth, response), 2), 0.67) # }) test_that("multiclass auc", { truth = factor(c("a", "b", "c")) prob = diag(3) colnames(prob) = levels(truth) expect_equal(mauc_aunu(truth, prob), 1) expect_equal(mauc_aunp(truth, prob), 1) expect_equal(mauc_au1u(truth, prob), 1) expect_equal(mauc_au1p(truth, prob), 1) expect_equal(mauc_mu(truth, prob), 1) auc(truth = factor(c("a", "nota", "nota")), prob = c(1, 0, 0), positive = "a") truth = ssample(c("a", "b", "c"), 100) prob = matrix(runif(300), ncol = 3) colnames(prob) = levels(truth) # having the same number of 'a', 'b', and 'c' gives the same 'n' and 'u' measures equalizer_prob = matrix(runif(900), ncol = 3) colnames(equalizer_prob) = levels(truth) equalizer_truth = unlist(list( truth, factor(truth, levels = c("b", "c", "a"), labels = c("a", "b", "c")), factor(truth, levels = c("c", "a", "b"), labels = c("a", "b", "c")) )) expect_equal(mauc_aunu(equalizer_truth, equalizer_prob), mauc_aunp(equalizer_truth, equalizer_prob)) expect_equal(mauc_au1u(equalizer_truth, equalizer_prob), mauc_au1p(equalizer_truth, equalizer_prob)) # having no information in prob gives measure 0.5 maxent_prob = rbind(prob, prob, prob) expect_equal(mauc_aunu(equalizer_truth, maxent_prob), 0.5) expect_equal(mauc_aunp(equalizer_truth, maxent_prob), 0.5) expect_equal(mauc_aunu(equalizer_truth, maxent_prob), 0.5) expect_equal(mauc_au1u(equalizer_truth, maxent_prob), 0.5) expect_equal(mauc_mu(equalizer_truth, maxent_prob), 0.5) # reversing prob gives 1 - auc expect_equal(mauc_aunu(truth, prob), 1 - mauc_aunu(truth, 1 - prob)) expect_equal(mauc_aunp(truth, prob), 1 - mauc_aunp(truth, 1 - prob)) expect_equal(mauc_au1u(truth, prob), 1 - mauc_au1u(truth, 1 - prob)) expect_equal(mauc_au1p(truth, prob), 1 - mauc_au1p(truth, 1 - prob)) expect_equal(mauc_mu(truth, prob), 1 - mauc_mu(truth, 1 - prob)) # manually calculate au1u, au1p compmat = sapply(levels(truth), function(t1) { sapply(levels(truth), function(t2) { if (t1 == t2) { return(0) } auc(factor(truth == t1)[truth %in% c(t1, t2)], prob[truth %in% c(t1, t2), t1], "TRUE") }) }) expect_equal(mauc_au1u(truth, prob), sum(compmat) / 6) expect_equal(mauc_au1p(truth, prob), sum(c(compmat + t(compmat)) * c(table(truth))) / 4 / length(truth)) # manually calculate aunu, aunp compvec = sapply(levels(truth), function(t1) { auc(factor(truth == t1), prob[, t1], "TRUE") }) expect_equal(mauc_aunu(truth, prob), mean(compvec)) expect_equal(mauc_aunp(truth, prob), sum(compvec * table(truth) / length(truth))) }) mlr3measures/tests/testthat.R0000644000176200001440000000024314472156310016023 0ustar liggesusersif (requireNamespace("testthat", quietly = TRUE)) { library("testthat") suppressPackageStartupMessages(library("mlr3measures")) test_check("mlr3measures") } mlr3measures/MD50000644000176200001440000001543715170431602013215 0ustar liggesusers9eba1b46613f75af7d32a47a71013fb7 *DESCRIPTION 89c0dc6e5204a2dd9cb46393a99bc5da *NAMESPACE d26ebc299728af5584d5f53e51aa5219 *NEWS.md cca4314a662242847f6e0b2b6813b297 *R/assertions.R a0d18ec83360c1a0bdd9320f3ee9aa4e *R/bibentries.R f906e0547eb23229f5dae661b78aaf63 *R/binary_auc.R bb4c992309ffd102639cea213c2b269e *R/binary_bbrier.R eb87ad9d09888d5c968cc72beb65ac5d *R/binary_dor.R 16a36bc2b8d936229b7b7e0d6433934f *R/binary_fbeta.R 89b5c2e575076f440b00b7b5848e4890 *R/binary_fdr.R eb207e6abd0348c8999373404d09f051 *R/binary_fn.R 75c63b41247bbf14f7c107f3a5c8ee7c *R/binary_fnr.R adf6234d3fbe920e34e17614a5f01b79 *R/binary_fomr.R 7fd2fb42fd7abb4630ebb47b098dc081 *R/binary_fp.R b4fcc96c29528dd99ff371a6eea35ddf *R/binary_fpr.R da6db5205d71d2c6ae72e56a46bd7e9d *R/binary_gmean.R 26d03319350f2f648c532f2a5922f2cd *R/binary_gpr.R e0195a1f85e6aee2649829dbc4914a0c *R/binary_npv.R 3efd9b4177b81b96f0b046a98ddcaa90 *R/binary_ppv.R 6de7afcb029f26f96562b863d5328cee *R/binary_prauc.R 1829359257f822abaa44878f35ce5b36 *R/binary_tn.R a1120961cb827c5c9e583d04be46b699 *R/binary_tnr.R f401f541ac87aa7c75a55aa782e546ff *R/binary_tp.R c08578f3c661402939b8859f12364e70 *R/binary_tpr.R 505e3d4ea30de86bb0e410ce9cbff72e *R/classif_acc.R c76bc29f616ac00621b6ce92521d50cd *R/classif_auc.R 689a780a497a55bcd12651c543a0347a *R/classif_bacc.R 3708ad78951f93b8b147b7602ec6d257 *R/classif_ce.R 4132d5e5d3e8ef77e67bac989c89d3df *R/classif_logloss.R a698abf3772f24da11466cd27fc2cfa6 *R/classif_mbrier.R 2ba43035d7787bfce6ff32c87dec2994 *R/classif_mcc.R acdbc2b63414bd0dce40f7e42196fdfd *R/classif_zero_one.R c0a79dbea8e19382e2310f5cc830765e *R/confusion_matrix.R a223f71e72cc8cc543affddc0daf9235 *R/helper.R 2ce458082356702f6c69633321eb8459 *R/measures.R 38eacecdded3eb574b71ad76a559e21e *R/regr_ae.R 8924120086d54e480cf8a7845b9e7555 *R/regr_ape.R 0d230666db8a534c062b506083b15d26 *R/regr_bias.R a1b5495e0483b033f675d78f34cdd514 *R/regr_ktau.R d4a656b4227bc86e5949d9df6518e874 *R/regr_linex.R ce32ea854e133069d347e41e4e5c09e4 *R/regr_mae.R 4eb0e3ccc80bc022338851d7b97f1a39 *R/regr_mape.R fa629d9614286abe7b4296dd004a53d5 *R/regr_maxae.R 7033c633251b1db9f9c6add7830bf21d *R/regr_maxse.R 55e284a3f932c369c9e00150d70fab16 *R/regr_medae.R 18b19f76228d7993d746a2517d492b73 *R/regr_medse.R f4f4105d6f0929006e0f9b66af23c530 *R/regr_mse.R aaf4d2ef2e217deb285e22bb75e57d7d *R/regr_msle.R 79f6c52f19d787fe73604ef55db7f6db *R/regr_pbias.R dfacc8043d613ee5bc6d744dd680dbe6 *R/regr_pinball.R aecfe5f5df2e3843bce8907a625595e6 *R/regr_rae.R 48536abf58ec26be8881e82b27f24e42 *R/regr_rmse.R 49bc2a80d9758b095cbd6d0cf212b776 *R/regr_rmsle.R 100e0aac77fe4ecfa5cb58505cc42a33 *R/regr_rrse.R 652a192509ec9fb09bd136de4218c645 *R/regr_rse.R 564a8f6e671b083de6f01a8e21fcee03 *R/regr_rsq.R 13b26d99746054b793faea6d6d9110b4 *R/regr_sae.R 9c9252952134887d51af3548bcae0abd *R/regr_se.R 4bd843729947d5de510b48ff28e30b0a *R/regr_sle.R e30e1fe69150080194e421227da310d0 *R/regr_smape.R f6715d70c1289b864b699170dd9f6869 *R/regr_srho.R 3dcbdde9b9ad9a5e2f8c50523c9ff0b8 *R/regr_sse.R c112eda49227f08ad498ed9c6978f4b5 *R/roxygen.R 57c634e24445d824afa0626411a52cd4 *R/similarity_jaccard.R 9954cce37d719b16fb300bdc5a2f8cb7 *R/similarity_phi.R cf268fe02889fc0be7e5d358e5f66abf *R/zzz.R 6df1c487d0d7b6e36b46edf92bb7eb0e *README.md f51ba7c8bc94d8cf1fd072b85c6e53ab *build/partial.rdb bd57e570737cc70d8b9cfd73bc0cb423 *man/acc.Rd 8d56710db1ff493f55b3ff556312e24f *man/ae.Rd 93d250e763b452f2f6d91ede0847e40b *man/ape.Rd 650a31cbcb1011566147b2bae753b3e4 *man/auc.Rd 722c42e74a0959a66aae31ad324b7537 *man/bacc.Rd a631d54eb80c944696fad19e08c767bb *man/bbrier.Rd 8cb9634c24a123ad4f7c1ccab8f8348e *man/bias.Rd 58e10343a09a5c9afdaaaf43cff4c34a *man/binary_params.Rd 83481817e5ad906738586e5bc4c8866c *man/ce.Rd a41441b22190cce7bb7fa00e5c5d477f *man/classif_params.Rd 683c880a43b221552a88bd84a93b4c7a *man/confusion_matrix.Rd 23fc3d55294b02f14f4949dcf3cea424 *man/dor.Rd d6b40e2c8029b900e335ed566efe9c5f *man/fbeta.Rd 8ccc67302d3ee1dda4862146870d1912 *man/fdr.Rd 8847178d92e213bbb40d5c6609e79e58 *man/fn.Rd 98fabbdcc89521f38fa9af62b6fdf8ea *man/fnr.Rd d2ad4b2f565ce19a2845c826e1f96701 *man/fomr.Rd f7756adda89e905cae011324eb968258 *man/fp.Rd 209efba0973d2ffd7ab0117ebe9b9e2d *man/fpr.Rd 0d9ab5c66fa0cb9a0ae355b4d25df26a *man/gmean.Rd 339b2f82dea7d0bf27a7b1e457154478 *man/gpr.Rd 2464f6131ae6f85dce0480c11fb90266 *man/jaccard.Rd d85d09ff05cc3db895272048b66c9669 *man/ktau.Rd 2830d0ec41fa190edf3b4685992ff798 *man/linex.Rd 8ef630876152f91992756552b8eac36c *man/logloss.Rd 83b939cadc2900a365e37b6f13fba2b6 *man/macros/cite.Rd af0d7a1ce23e09bd158b10c65d74926e *man/mae.Rd 5573701d197126d35910ebfb98fe2d81 *man/mape.Rd c8ba0f99cf861ec9dde71937f93ddf98 *man/mauc_aunu.Rd e5ba676e19929bbc62b413e230168400 *man/maxae.Rd 1df7ac4bf17439b9ddb7742ef180b964 *man/maxse.Rd 5f1f9481901e7f35718182419b1aafa1 *man/mbrier.Rd 042012d89e8cf49c9e48a484c469379d *man/mcc.Rd 5b564868e1d279d8cf6299ce439c6f14 *man/measures.Rd e07678de24264af5a767829ee1bc0db1 *man/medae.Rd 290f44ef377555c158ed9299361557ef *man/medse.Rd ca0c03c9c50061405de204234ffff048 *man/mlr3measures-package.Rd cc6f704e749f651bceb053e03b3bdf0e *man/mse.Rd b78043668df755dcb458f3752aeaec24 *man/msle.Rd 44faf0ac944ab1aa9929523fc3d3843f *man/npv.Rd 3fcb35491efc5ed2edc9d2d4457761dd *man/obs_logloss.Rd 1b32edad166a5990b45008e94d7d7878 *man/pbias.Rd 08bb8100d23dfa80e21562e4caa4dde2 *man/phi.Rd 6c9fe736561150b8f8935586bbead97a *man/pinball.Rd c1cdd27d2e71a1819d4fc34d885ff3b9 *man/ppv.Rd d5fc7f3b8ff0a7c3d29514ded09a8fd0 *man/prauc.Rd fb90b3c5f70aacc23d6a59b94e5b57c0 *man/rae.Rd a9236a29c2e51f53244998afbcf29009 *man/regr_params.Rd 1ba63241b96e246f8dac828a216c960f *man/rmse.Rd eda039aaa566e5c609af2faa22ab9694 *man/rmsle.Rd 2135c1891df81a3c5ca57dc937dfdc56 *man/rrse.Rd 291d614976afaba23b8d3764b5ea232c *man/rse.Rd e42c9116c97f1d5e8fcd1741c44e4bb3 *man/rsq.Rd 3916e5809624c24e25bc797655afedd7 *man/sae.Rd 0a95de71f37b56b772b11b8bc6b82813 *man/se.Rd d024d4a33750ec1c1e0d09d32f994ddd *man/se_binary.Rd cd960cfe4aef13b615a0bcef2d26d960 *man/similarity_params.Rd 13c6f4a7304d379332500d18f00a290e *man/sle.Rd dbb5bfad273caebebc66cc43ff2746a0 *man/smape.Rd 647e582612930b79ee1fd4dd560f4793 *man/srho.Rd cc78c9de335a2191caaccbfb29747235 *man/sse.Rd 413fee4717832cdbbb2e44902551a3de *man/tn.Rd 0183486a2ae39f306397166870bfa75b *man/tnr.Rd 83cc8cd16ffb3d85146fb8b89057142b *man/tp.Rd 2c5b69be49a9b9320932e6b6ca3c152a *man/tpr.Rd 21d020ea69c37f996fbfe16eecfb0a9c *man/zero_one.Rd a9c1326d15dbe72b8129b3f9fd36d095 *tests/testthat.R 0f7d8a107eaacc0c7970923974a66cf1 *tests/testthat/helper.R 7e55797ee4d53d7ef6cd613c0b51279e *tests/testthat/test_binary.R f2179caa3f4e0027b1495c2904bbaf47 *tests/testthat/test_binary_bbrier.R 85e2f1f30950f73dbabbb56459044637 *tests/testthat/test_classif.R a192827e86e2ce6a31b364e465039eb3 *tests/testthat/test_classif_logloss.R 69d6d685fb5cabbb93c7864f61ef12e7 *tests/testthat/test_regr.R 8056604c17f23770604b4474a78b476b *tests/testthat/test_similarity.R mlr3measures/R/0000755000176200001440000000000015170376754013114 5ustar liggesusersmlr3measures/R/binary_tp.R0000644000176200001440000000135715170123773015223 0ustar liggesusers#' @title True Positives #' #' @details #' This measure counts the true positives, i.e. the number of #' predictions correctly indicating a positive class label. This is sometimes also called a "hit". #' #' @templateVar mid tp #' @template binary_template #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export tp = function(truth, response, positive, sample_weights = NULL, ...) { assert_binary(truth, response = response, positive = positive) tp_cm(cm(truth, response, positive, sample_weights)) } tp_cm = function(m, na_value = NaN) { m[1L, 1L] } #' @include measures.R add_measure(tp, "True Positives", "binary", 0, Inf, FALSE) mlr3measures/R/binary_fnr.R0000644000176200001440000000160515170123773015361 0ustar liggesusers#' @title False Negative Rate #' #' @details #' The False Negative Rate is defined as \deqn{ #' \frac{\mathrm{FN}}{\mathrm{TP} + \mathrm{FN}}. #' }{ #' FN / (TP + FN). #' } #' Also know as "miss rate". #' #' @templateVar mid fnr #' @template binary_template #' #' @details #' This measure is undefined if TP + FN = 0. #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export fnr = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) fnr_cm(cm(truth, response, positive, sample_weights), na_value) } fnr_cm = function(m, na_value = NaN) { div(m[2L, 1L], sum(m[, 1L]), na_value) } #' @include measures.R add_measure(fnr, "False Negative Rate", "binary", 0, 1, TRUE) mlr3measures/R/regr_rse.R0000644000176200001440000000166315170376754015055 0ustar liggesusers#' @title Relative Squared Error #' #' @details #' The Relative Squared Error is defined as \deqn{ #' \frac{\sum_{i=1}^n \left( t_i - r_i \right)^2}{\sum_{i=1}^n \left( t_i - \bar{t} \right)^2}, #' }{ #' sum((t - r)^2) / sum((t - mean(t))^2), #' } #' where \eqn{\bar{t} = \sum_{i=1}^n t_i}. #' #' Can be interpreted as squared error of the predictions relative to a naive model predicting the mean. #' #' This measure is undefined for constant \eqn{t}. #' #' @templateVar mid rse #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export rse = function(truth, response, na_value = NaN, ...) { .Deprecated(msg = "rse is deprecated") assert_regr(truth, response = response, na_value = na_value) v = var(truth) if (v < TOL) { return(na_value) } sum(.se(truth, response)) / (v * (length(truth) - 1L)) } #' @include measures.R add_measure(rse, "Relative Squared Error", "regr", 0, Inf, TRUE) mlr3measures/R/regr_rmsle.R0000644000176200001440000000160614661131133015363 0ustar liggesusers#' @title Root Mean Squared Log Error #' #' @details #' The Root Mean Squared Log Error is defined as \deqn{ #' \sqrt{\frac{1}{n} \sum_{i=1}^n w_i \left( \ln (1 + t_i) - \ln (1 + r_i) \right)^2}, #' }{ #' sqrt(weighted.mean((log(1 + t) - log(1 + r))^2, w)), #' } #' where \eqn{w_i} are normalized sample weights. #' #' This measure is undefined if any element of \eqn{t} or \eqn{r} is less than or equal to \eqn{-1}. #' #' @templateVar mid rmsle #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export rmsle = function(truth, response, sample_weights = NULL, na_value = NaN, ...) { assert_regr(truth, response = response, na_value = na_value) if (min(truth, response) <= -1) { return(na_value) } sqrt(wmean(.sle(truth, response), sample_weights)) } #' @include measures.R add_measure(rmsle, "Root Mean Squared Log Error", "regr", 0, Inf, TRUE) mlr3measures/R/regr_rmse.R0000644000176200001440000000134515170376754015227 0ustar liggesusers#' @title Root Mean Squared Error #' #' @details #' The Root Mean Squared Error is defined as \deqn{ #' \sqrt{\frac{1}{n} \sum_{i=1}^n w_i \left( t_i - r_i \right)^2}, #' }{ #' sqrt(weighted.mean((t - r)^2, w)), #' } #' where \eqn{w_i} are normalized sample weights. #' #' @templateVar mid rmse #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export rmse = function(truth, response, sample_weights = NULL, ...) { assert_regr(truth, response = response) sqrt(wmean(.se(truth, response), sample_weights)) } #' @include measures.R add_measure( rmse, "Root Mean Squared Error", "regr", 0, Inf, TRUE, obs_loss = "se", trafo = list(fn = sqrt, deriv = function(x) 0.5 / sqrt(x)) ) mlr3measures/R/binary_fbeta.R0000644000176200001440000000302515170123773015653 0ustar liggesusers#' @title F-beta Score #' #' @details #' With \eqn{P} as [precision()] and \eqn{R} as [recall()], the F-beta Score is defined as \deqn{ #' (1 + \beta^2) \frac{P \cdot R}{(\beta^2 P) + R}. #' }{ #' (1 + beta^2) * (P*R) / ((beta^2 * P) + R). #' } #' It measures the effectiveness of retrieval with respect to a user who attaches \eqn{\beta}{beta} times #' as much importance to recall as precision. #' For \eqn{\beta = 1}{beta = 1}, this measure is called "F1" score. #' #' @templateVar mid fbeta #' @template binary_template #' #' @details #' This measure is undefined if [precision] or [recall] is undefined, i.e. TP + FP = 0 or TP + FN = 0. #' #' @references #' `r format_bib("rijsbergen_1979", "goutte_2005")` #' #' @inheritParams binary_params #' @param beta (`numeric(1)`)\cr #' Parameter to give either precision or recall more weight. #' Default is `1`, resulting in balanced weights. #' @template binary_example #' @export fbeta = function(truth, response, positive, sample_weights = NULL, beta = 1, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) assert_number(beta, lower = 0) fbeta_cm(cm(truth, response, positive, sample_weights), beta, na_value) } fbeta_cm = function(m, beta = 1, na_value = NaN) { if (m[1L, 1L] == 0L && (m[1L, 2L] == 0L || m[2L, 1L] == 0L)) { return(na_value) } beta2 = beta^2 nom = (1 + beta2) * m[1L, 1L] nom / (nom + beta2 * m[2L, 1L] + m[1L, 2L]) } #' @include measures.R add_measure(fbeta, "F-beta score", "binary", 0, 1, FALSE) mlr3measures/R/binary_bbrier.R0000644000176200001440000000245715111057476016050 0ustar liggesusers#' @title Binary Brier Score #' #' @details #' The Binary Brier Score is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i (I_i - p_i)^2, #' }{ #' weighted.mean(((t == positive) - p)^2, w), #' } #' where \eqn{w_i} are the sample weights, #' and \eqn{I_{i}} is 1 if observation \eqn{x_i} belongs to the positive class, and 0 otherwise. #' #' Note that this (more common) definition of the Brier score is equivalent to the #' original definition of the multi-class Brier score (see [mbrier()]) divided by 2. #' #' @templateVar mid bbrier #' @template binary_template #' #' @references #' \url{https://en.wikipedia.org/wiki/Brier_score} #' #' `r format_bib("brier_1950")` #' #' @inheritParams binary_params #' @template binary_example #' @export bbrier = function(truth, prob, positive, sample_weights = NULL, ...) { assert_binary(truth, prob = prob, positive = positive) wmean(.se(truth == positive, prob), sample_weights) } #' @include measures.R add_measure(bbrier, "Binary Brier Score", "binary", 0, 1, TRUE, obs_loss = "se_binary") #' @title Binary Squared Error #' #' @description #' Observation-wise loss function for [bbrier()]. #' #' @export #' @keywords internal se_binary = function(truth, prob, positive, ...) { assert_binary(truth, prob = prob, positive = positive) ((truth == positive) - prob)^2 } mlr3measures/R/binary_gmean.R0000644000176200001440000000177515170123773015673 0ustar liggesusers#' @title Geometric Mean of Recall and Specificity #' #' @details #' Calculates the geometric mean of [recall()] R and [specificity()] S as \deqn{ #' \sqrt{\mathrm{R} \cdot \mathrm{S}}. #' }{ #' sqrt(R * S) #' } #' #' @templateVar mid gmean #' @template binary_template #' #' @details #' This measure is undefined if recall or specificity is undefined, i.e. if TP + FN = 0 or if FP + TN = 0. #' #' @references #' `r format_bib("he_2009")` #' #' @inheritParams binary_params #' @template binary_example #' @export gmean = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) gmean_cm(cm(truth, response, positive, sample_weights), na_value) } gmean_cm = function(m, na_value = NaN) { cs = colSums(m) if (any(cs == 0L)) { return(na_value) } sqrt(prod(diag(m) / colSums(m))) } #' @include measures.R add_measure(gmean, "Geometric Mean of Recall and Specificity", "binary", 0, 1, FALSE) mlr3measures/R/measures.R0000644000176200001440000000704415170376754015070 0ustar liggesusers#' @title Measure Registry #' #' @description #' The [environment()] `measures` keeps track of all measures in this package. #' It stores meta information such as minimum, maximum or if the #' measure must be minimized or maximized. #' The following information is available for each measure: #' * `id`: Name of the measure. #' * `title`: Short descriptive title. #' * `type`: `"binary"` for binary classification, `"classif"` for binary or multi-class classification, #' `"regr"` for regression and `"similarity"` for similarity measures. #' * `lower`: lower bound. #' * `upper`: upper bound. #' * `predict_type`: prediction type the measure operates on. #' `"response"` corresponds to class labels for classification and the numeric response for regression. #' `"prob"` corresponds to class probabilities, provided as a matrix with class labels as column names. #' `"se"` corresponds to the vector of predicted standard errors for regression. #' * `minimize`: If `TRUE` or `FALSE`, the objective is to minimize or maximize the measure, respectively. #' Can also be `NA`. #' * `obs_loss`: Name of the function which is called to calculate the (unaggregated) loss per observation. #' * `trafo`: Optional `list()` of length 2, containing a transformation `"fn"` and its derivative `"deriv"`. #' * `aggregated`: If `TRUE`, this function aggregates the losses to a single numeric value. #' Otherwise, a vector of losses is returned. #' * `sample_weights`: If `TRUE`, it is possible calculate a weighted measure. #' #' @export #' @examples #' names(measures) #' measures$tpr #' as.data.frame(measures) measures = new.env(parent = emptyenv()) class(measures) = c("MeasureEnv", class(measures)) # adds items to registry add_measure = function( obj, title, type, lower, upper, minimize, obs_loss = NA_character_, trafo = NA_character_, aggregated = TRUE ) { id = deparse(substitute(obj)) ptype = intersect(names(formals(obj)), c("response", "prob", "se")) if (length(ptype) == 0L) { ptype = NA_character_ } if (!identical(trafo, NA_character_)) { assert_list(trafo, types = "function", len = 2L) assert_permutation(names(trafo), c("fn", "deriv")) assert_true(!is.na(obs_loss)) } assign( id, list( id = id, title = assert_string(title), type = assert_choice(type, c("binary", "classif", "regr", "similarity")), lower = assert_number(lower), upper = assert_number(upper), predict_type = ptype, minimize = assert_flag(minimize, na.ok = TRUE), obs_loss = assert_string(obs_loss, na.ok = TRUE), aggregated = assert_flag(aggregated), sample_weights = "sample_weights" %in% names(formals(obj)), trafo = trafo ), envir = measures ) } #' @export print.MeasureEnv = function(x, ...) { cat(sprintf("Environment with %i measures, see `names(measures)` and `as.data.frame(measures)`\n", length(x))) } #' @export as.data.frame.MeasureEnv = function(x, ...) { data.frame( id = vapply(x, function(x) x$id, NA_character_, USE.NAMES = FALSE), title = vapply(x, function(x) x$title, NA_character_, USE.NAMES = FALSE), type = vapply(x, function(x) x$type, NA_character_, USE.NAMES = FALSE), lower = vapply(x, function(x) x$lower, NA_real_, USE.NAMES = FALSE), upper = vapply(x, function(x) x$upper, NA_real_, USE.NAMES = FALSE), minimize = vapply(x, function(x) x$minimize, NA, USE.NAMES = FALSE), obs_loss = vapply(x, function(x) !is.na(x$obs_loss), NA, USE.NAMES = FALSE), sample_weights = vapply(x, function(x) x$sample_weights, NA, USE.NAMES = FALSE) ) } mlr3measures/R/binary_tnr.R0000644000176200001440000000200215170123773015367 0ustar liggesusers#' @title True Negative Rate #' #' @details #' The True Negative Rate is defined as \deqn{ #' \frac{\mathrm{TN}}{\mathrm{FP} + \mathrm{TN}}. #' }{ #' TN / (FP + TN). #' } #' Also know as "specificity" or "selectivity". #' #' @templateVar mid tnr #' @template binary_template #' #' @details #' This measure is undefined if FP + TN = 0. #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export tnr = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) tnr_cm(cm(truth, response, positive, sample_weights), na_value) } tnr_cm = function(m, na_value = NaN) { div(m[2L, 2L], sum(m[, 2L]), na_value) } #' @export #' @rdname tnr specificity = tnr #' @include measures.R add_measure(tnr, "True Negative Rate", "binary", 0, 1, FALSE) add_measure(specificity, "Specificity", "binary", 0, 1, FALSE) mlr3measures/R/binary_fpr.R0000644000176200001440000000164015170123773015362 0ustar liggesusers#' @title False Positive Rate #' #' @details #' The False Positive Rate is defined as \deqn{ #' \frac{\mathrm{FP}}{\mathrm{FP} + \mathrm{TN}}. #' }{ #' FP / (FP + TN). #' } #' Also know as fall out or probability of false alarm. #' #' @templateVar mid fpr #' @template binary_template #' #' @details #' This measure is undefined if FP + TN = 0. #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export fpr = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) fpr_cm(cm(truth, response, positive, sample_weights), na_value) } fpr_cm = function(m, na_value = NaN) { div(m[1L, 2L], sum(m[, 2L]), na_value) } #' @include measures.R add_measure(fpr, "False Positive Rate", "binary", 0, 1, TRUE) mlr3measures/R/regr_ape.R0000644000176200001440000000114514661131133015004 0ustar liggesusers#' @title Absolute Percentage Error (per observation) #' #' @details #' Calculates the per-observation absolute percentage error as \deqn{ #' \left| \frac{ t_i - r_i}{t_i} \right|. #' }{ #' abs((t - r) / t). #' } #' #' @templateVar mid ape #' @template regr_template #' @inheritParams regr_params #' #' @export ape = function(truth, response, ...) { assert_regr(truth, response) .ape(truth, response) } .ape = function(truth, response) { abs((truth - response) / truth) } #' @include measures.R add_measure(ape, "Absolute Percentage Error (per observation)", "regr", 0, Inf, TRUE, aggregated = FALSE) mlr3measures/R/regr_srho.R0000644000176200001440000000111114472156310015207 0ustar liggesusers#' @title Spearman's rho #' #' @details #' Spearman's rho is defined as Spearman's rank correlation coefficient between truth and response. #' Calls [stats::cor()] with `method` set to `"spearman"`. #' #' @templateVar mid srho #' @template regr_template #' #' @references #' `r format_bib("rosset_2006")` #' #' @inheritParams regr_params #' @template regr_example #' @export srho = function(truth, response, ...) { assert_regr(truth, response = response) cor(truth, response, method = "spearman") } #' @include measures.R add_measure(srho, "Spearman's rho", "regr", -1, 1, FALSE) mlr3measures/R/roxygen.R0000644000176200001440000000642714472156310014727 0ustar liggesusers#' @title Regression Parameters #' @name regr_params #' #' @param truth (`numeric()`)\cr #' True (observed) values. #' Must have the same length as `response`. #' @param response (`numeric()`)\cr #' Predicted response values. #' Must have the same length as `truth`. #' @param sample_weights (`numeric()`)\cr #' Vector of non-negative and finite sample weights. #' Must have the same length as `truth`. #' The vector gets automatically normalized to sum to one. #' Defaults to equal sample weights. #' @param na_value (`numeric(1)`)\cr #' Value that should be returned if the measure is not defined for the input #' (as described in the note). Default is `NaN`. #' @param ... (`any`)\cr #' Additional arguments. Currently ignored. #' @keywords internal NULL #' @title Binary Classification Parameters #' @name binary_params #' #' @param truth (`factor()`)\cr #' True (observed) labels. #' Must have the exactly same two levels and the same length as `response`. #' @param response (`factor()`)\cr #' Predicted response labels. #' Must have the exactly same two levels and the same length as `truth`. #' @param prob (`numeric()`)\cr #' Predicted probability for positive class. #' Must have exactly same length as `truth`. #' @param positive (`character(1))`\cr #' Name of the positive class. #' @param sample_weights (`numeric()`)\cr #' Vector of non-negative and finite sample weights. #' Must have the same length as `truth`. #' The vector gets automatically normalized to sum to one. #' Defaults to equal sample weights. #' @param na_value (`numeric(1)`)\cr #' Value that should be returned if the measure is not defined for the input #' (as described in the note). Default is `NaN`. #' @param ... (`any`)\cr #' Additional arguments. Currently ignored. #' @keywords internal NULL #' @title Classification Parameters #' @name classif_params #' #' @param truth (`factor()`)\cr #' True (observed) labels. #' Must have the same levels and length as `response`. #' @param response (`factor()`)\cr #' Predicted response labels. #' Must have the same levels and length as `truth`. #' @param prob (`matrix()`)\cr #' Matrix of predicted probabilities, each column is a vector of probabilities for a #' specific class label. #' Columns must be named with levels of `truth`. #' @param sample_weights (`numeric()`)\cr #' Vector of non-negative and finite sample weights. #' Must have the same length as `truth`. #' The vector gets automatically normalized to sum to one. #' Defaults to equal sample weights. #' @param na_value (`numeric(1)`)\cr #' Value that should be returned if the measure is not defined for the input #' (as described in the note). Default is `NaN`. #' @param ... (`any`)\cr #' Additional arguments. Currently ignored. #' @keywords internal NULL #' @title Similarity Parameters #' @name similarity_params #' #' @param sets (`list()`)\cr #' List of character or integer vectors. #' `sets` must have at least 2 elements. #' @param p (`integer(1)`)\cr #' Total number of possible elements. #' @param na_value (`numeric(1)`)\cr #' Value that should be returned if the measure is not defined for the input #' (as described in the note). Default is `NaN`. #' @param ... (`any`)\cr #' Additional arguments. Currently ignored. #' @keywords internal NULL mlr3measures/R/regr_pbias.R0000644000176200001440000000140415056012322015330 0ustar liggesusers#' @title Percent Bias #' #' @details #' The Percent Bias is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i \frac{\left( r_i - t_i \right)}{\left| t_i \right|}, #' }{ #' weighted.mean((r - t) / abs(t), w), #' } #' where \eqn{w_i} are normalized sample weights. #' Good predictions score close to 0. #' #' @templateVar mid pbias #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export pbias = function(truth, response, sample_weights = NULL, na_value = NaN, ...) { assert_regr(truth, response = response, na_value = na_value) if (any(abs(truth) < TOL)) { return(na_value) } wmean((response - truth) / abs(truth), sample_weights) } #' @include measures.R add_measure(pbias, "Percent Bias", "regr", -Inf, Inf, NA) mlr3measures/R/regr_medae.R0000644000176200001440000000103214661131133015305 0ustar liggesusers#' @title Median Absolute Error #' #' @details #' The Median Absolute Error is defined as \deqn{ #' \mathop{\mathrm{median}} \left| t_i - r_i \right|. #' }{ #' median(abs(t - r)). #' } #' #' @templateVar mid medae #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export medae = function(truth, response, ...) { assert_regr(truth, response = response) median(.ae(truth, response)) } #' @include measures.R add_measure(medae, "Median Absolute Error", "regr", 0, Inf, TRUE, obs_loss = "ae") mlr3measures/R/regr_mse.R0000644000176200001440000000116214661131133015022 0ustar liggesusers#' @title Mean Squared Error #' #' @details #' The Mean Squared Error is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i \left( t_i - r_i \right)^2, #' }{ #' weighted.mean((t - r)^2, w), #' } #' where \eqn{w_i} are normalized sample weights. #' #' @templateVar mid mse #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export mse = function(truth, response, sample_weights = NULL, ...) { assert_regr(truth, response = response) wmean(.se(truth, response), sample_weights) } #' @include measures.R add_measure(mse, "Mean Squared Error", "regr", 0, Inf, TRUE, obs_loss = "se") mlr3measures/R/binary_fdr.R0000644000176200001440000000155315170123773015351 0ustar liggesusers#' @title False Discovery Rate #' #' @details #' The False Discovery Rate is defined as \deqn{ #' \frac{\mathrm{FP}}{\mathrm{TP} + \mathrm{FP}}. #' }{ #' FP / (TP + FP). #' } #' #' @templateVar mid fdr #' @template binary_template #' #' @details #' This measure is undefined if TP + FP = 0. #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export fdr = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) fdr_cm(cm(truth, response, positive, sample_weights), na_value) } fdr_cm = function(m, na_value = NaN) { div(m[1L, 2L], sum(m[1L, ]), na_value) } #' @include measures.R add_measure(fdr, "False Discovery Rate", "binary", 0, 1, TRUE) mlr3measures/R/helper.R0000644000176200001440000000316315170376754014521 0ustar liggesusersTOL = sqrt(.Machine$double.eps) # simple division, but ensures that `na_value` is returned # if the denominator is < TOL div = function(nominator, denominator, na_value) { if (abs(denominator) < TOL) { na_value } else { nominator / denominator } } wmean = function(x, w) { # a better stats::weighted.mean if (is.null(w)) { return(mean(x)) } assert_numeric(w, lower = 0, finite = TRUE, any.missing = FALSE, len = length(x)) if (all(abs(w) < TOL)) { stop("All sample weights are 0") } sum(x * (w / sum(w))) } wsum = function(x, w) { # sum(w * x) that asserts w and accepts NULL if (is.null(w)) { return(sum(x)) } assert_numeric(w, lower = 0, finite = TRUE, any.missing = FALSE, len = length(x)) sum(x * w) } # confusion matrix cm = function(truth, response, positive = NULL, sample_weights = NULL) { if (!is.null(positive)) { truth = relevel(truth, positive) response = relevel(response, positive) } if (is.null(sample_weights)) { return(table(response, truth, useNA = "ifany")) } xtabs(sample_weights ~ response + truth, addNA = TRUE) } # used in roxygen templates format_range = function(item) { l = item$lower u = item$upper str = sprintf( "%s%s, %s%s", if (is.finite(l)) "[" else "(", if (is.finite(l)) c(l, l) else c("-\\infty", "-Inf"), if (is.finite(u)) c(u, u) else c("\\infty", "Inf"), if (is.finite(u)) "]" else ")" ) paste0("\\eqn{", str[1L], "}{", str[2L], "}") } map_pairwise = function(x, .fun, ...) { .fun = match.fun(.fun) combn(x, m = 2L, function(.pair, ...) .fun(.pair[[1L]], .pair[[2L]], ...), simplify = TRUE, ...) } mlr3measures/R/binary_dor.R0000644000176200001440000000166515170123773015366 0ustar liggesusers#' @title Diagnostic Odds Ratio #' #' @details #' The Diagnostic Odds Ratio is defined as \deqn{ #' \frac{\mathrm{TP}/\mathrm{FP}}{\mathrm{FN}/\mathrm{TN}}. #' }{ #' (TP/FP) / (FN/TN). #' } #' #' @templateVar mid dor #' @template binary_template #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @details #' This measure is undefined if FP = 0 or FN = 0. #' #' @inheritParams binary_params #' @template binary_example #' @export dor = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) dor_cm(cm(truth, response, positive, sample_weights), na_value) } dor_cm = function(m, na_value = NaN) { storage.mode(m) = "double" div(m[1L, 1L] * m[2L, 2L], m[1L, 2L] * m[2L, 1L], na_value) } #' @include measures.R add_measure(dor, "Diagnostic Odds Ratio", "binary", 0, Inf, FALSE) mlr3measures/R/regr_sle.R0000644000176200001440000000114414472156310015025 0ustar liggesusers#' @title Squared Log Error (per observation) #' #' @description #' Calculates the per-observation squared error as \deqn{ #' \left( \ln (1 + t_i) - \ln (1 + r_i) \right)^2. #' }{ #' (log(1 + t) - log(1 + r))^2. #' } #' #' @templateVar mid sle #' @template regr_template #' @inheritParams regr_params #' #' @export sle = function(truth, response, ...) { assert_regr(truth, response) .sle(truth, response) } .sle = function(truth, response) { (log1p(truth) - log1p(response))^2 } #' @include measures.R add_measure(sle, "Squared Log Error (per observation)", "regr", 0, Inf, TRUE, aggregated = FALSE) mlr3measures/R/regr_msle.R0000644000176200001440000000156614661131133015206 0ustar liggesusers#' @title Mean Squared Log Error #' #' @details #' The Mean Squared Log Error is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i \left( \ln (1 + t_i) - \ln (1 + r_i) \right)^2, #' }{ #' weighted.mean((log(1 + t) - log(1 + r))^2, weights), #' } #' where \eqn{w_i} are normalized sample weights. #' This measure is undefined if any element of \eqn{t} or \eqn{r} is less than or equal to \eqn{-1}. #' #' @templateVar mid msle #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export msle = function(truth, response, sample_weights = NULL, na_value = NaN, ...) { assert_regr(truth, response = response, na_value = na_value) if (min(truth, response) <= -1) { return(na_value) } wmean(.sle(truth, response), sample_weights) } #' @include measures.R add_measure(msle, "Mean Squared Log Error", "regr", 0, Inf, TRUE, obs_loss = "sle") mlr3measures/R/classif_acc.R0000644000176200001440000000134715056012454015461 0ustar liggesusers#' @title Classification Accuracy #' #' @details #' The Classification Accuracy is defined as #' \deqn{ #' \sum_{i=1}^n w_i \mathbf{1} \left( t_i = r_i \right), #' }{sum(wi * 1(ti = ri))} #' where \eqn{w_i} are weights normalized to sum to 1 for all observations \eqn{x_i}. #' #' @templateVar mid acc #' @template classif_template #' #' @inheritParams classif_params #' @template classif_example #' @export acc = function(truth, response, sample_weights = NULL, ...) { assert_classif(truth, response = response) wmean(truth == response, sample_weights) } acc_cm = function(m, na_value = NaN) { sum(diag(m)) / sum(m) } #' @include measures.R add_measure(acc, "Classification Accuracy", "classif", 0, 1, FALSE, obs_loss = "one_zero") mlr3measures/R/binary_ppv.R0000644000176200001440000000204515170123773015400 0ustar liggesusers#' @title Positive Predictive Value #' #' @details #' The Positive Predictive Value is defined as \deqn{ #' \frac{\mathrm{TP}}{\mathrm{TP} + \mathrm{FP}}. #' }{ #' TP / (TP + FP). #' } #' Also know as "precision". #' #' @templateVar mid ppv #' @template binary_template #' #' @details #' This measure is undefined if TP + FP = 0. #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' `r format_bib("goutte_2005")` #' #' #' @inheritParams binary_params #' @template binary_example #' @export ppv = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) ppv_cm(cm(truth, response, positive, sample_weights), na_value) } ppv_cm = function(m, na_value = NaN) { div(m[1L, 1L], sum(m[1L, ]), na_value) } #' @export #' @rdname ppv precision = ppv #' @include measures.R add_measure(ppv, "Positive Predictive Value", "binary", 0, 1, FALSE) add_measure(precision, "Precision", "binary", 0, 1, FALSE) mlr3measures/R/regr_rsq.R0000644000176200001440000000177515056260067015065 0ustar liggesusers#' @title R Squared #' #' @details #' R Squared is defined as \deqn{ #' 1 - \frac{\sum_{i=1}^n \left( t_i - r_i \right)^2}{\sum_{i=1}^n \left( t_i - \bar{t} \right)^2}, #' }{ #' 1 - sum((t - r)^2) / sum((t - mean(t))^2), #' } #' where \eqn{\bar{t} = \sum_{i=1}^n t_i}. #' #' Also known as coefficient of determination or explained variation. #' Subtracts the [rse()] from 1, hence it compares the squared error of #' the predictions relative to a naive model predicting the mean. #' #' This measure is undefined for constant \eqn{t}. #' #' @templateVar mid rsq #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export rsq = function(truth, response, na_value = NaN, ...) { .Deprecated(msg = "rsq is deprecated") assert_regr(truth, response = response, na_value = na_value) v = var(truth) if (v < TOL) { return(na_value) } 1 - sum(.se(truth, response)) / (v * (length(truth) - 1L)) } #' @include measures.R add_measure(rsq, "R Squared", "regr", -Inf, 1, FALSE) mlr3measures/R/regr_mape.R0000644000176200001440000000157314661131133015166 0ustar liggesusers#' @title Mean Absolute Percent Error #' #' @details #' The Mean Absolute Percent Error is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i \left| \frac{ t_i - r_i}{t_i} \right|, #' }{ #' weighted.mean(abs((t - r) / t), w), #' } #' where \eqn{w_i} are normalized sample weights. #' #' @details #' This measure is undefined if any element of \eqn{t} is \eqn{0}. #' #' @templateVar mid mape #' @template regr_template #' #' @references #' `r format_bib("de_myttenaere_2016")` #' #' @inheritParams regr_params #' @template regr_example #' @export mape = function(truth, response, sample_weights = NULL, na_value = NaN, ...) { assert_regr(truth, response = response) if (any(abs(truth) < TOL)) { return(na_value) } wmean(.ape(truth, response), sample_weights) } #' @include measures.R add_measure(mape, "Mean Absolute Percent Error", "regr", 0, Inf, TRUE, obs_loss = "ape") mlr3measures/R/binary_gpr.R0000644000176200001440000000177615170376754015406 0ustar liggesusers#' @title Geometric Mean of Precision and Recall #' #' @details #' Calculates the geometric mean of [precision()] P and [recall()] R as \deqn{ #' \sqrt{\mathrm{P} \cdot \mathrm{R}}. #' }{ #' sqrt(P * R) #' } #' #' @templateVar mid gpr #' @template binary_template #' #' @details #' This measure is undefined if precision or recall is undefined, i.e. if TP + FP = 0 or if TP + FN = 0. #' #' @references #' `r format_bib("he_2009")` #' #' @inheritParams binary_params #' @template binary_example #' @export gpr = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) gpr_cm(cm(truth, response, positive, sample_weights), na_value) } gpr_cm = function(m, na_value = NaN) { sums = c(sum(m[1L, ]), sum(m[, 1L])) if (any(sums == 0L)) { return(na_value) } sqrt(prod(m[1L, 1L] / sums)) } #' @include measures.R add_measure(gpr, "Geometric Mean of Precision and Recall", "binary", 0, 1, FALSE) mlr3measures/R/zzz.R0000644000176200001440000000070715170376754014100 0ustar liggesusers#' @importFrom utils head combn #' @importFrom stats cor median var relevel contr.treatment xtabs ave #' @import checkmate "_PACKAGE" # nolint next .onAttach = function(libname, pkgname) { packageStartupMessage(paste( "In order to avoid name clashes, do not attach 'mlr3measures'.", "Instead, only load the namespace with `requireNamespace(\"mlrmeasures\")`", "and access the measures directly via `::`, e.g. `mlr3measures::auc()`." )) } mlr3measures/R/similarity_phi.R0000644000176200001440000000263214472156310016254 0ustar liggesusers#' @title Phi Coefficient Similarity #' #' @details #' The Phi Coefficient is defined as the Pearson correlation between the binary #' representation of two sets \eqn{A} and \eqn{B}. #' The binary representation for \eqn{A} is a logical vector of #' length \eqn{p} with the i-th element being 1 if the corresponding #' element is in \eqn{A}, and 0 otherwise. #' #' If more than two sets are provided, the mean of all pairwise scores #' is calculated. #' #' @references #' `r format_bib("nogueira_2016", "bommert_2017", "stabm")` #' #' @templateVar mid phi #' @template similarity_template #' #' @details #' This measure is undefined if one set contains none or all possible elements. #' #' @inheritParams similarity_params #' @export #' @examples #' set.seed(1) #' sets = list( #' sample(letters[1:3], 1), #' sample(letters[1:3], 2) #' ) #' phi(sets, p = 3) phi = function(sets, p, na_value = NaN, ...) { sets = assert_sets(sets) assert_count(p, positive = TRUE) if (any(lengths(sets) %in% c(0L, p))) { return(na_value) } omega = unique(unlist(sets, use.names = FALSE)) if (p < length(omega)) { stop("Number of observed elements in sets exceeds 'p'") } omega = c(omega, rep(NA, p - length(omega))) scores = map_pairwise(sets, function(A, B) { cor(omega %in% A, omega %in% B) }) mean(scores) } #' @include measures.R add_measure(phi, "Phi Coefficient Similarity", "similarity", -1, 1, FALSE) mlr3measures/R/classif_mcc.R0000644000176200001440000000511015170376754015502 0ustar liggesusers#' @title Matthews Correlation Coefficient #' #' @details #' In the binary case, the Matthews Correlation Coefficient is defined as \deqn{ #' \frac{\mathrm{TP} \cdot \mathrm{TN} - \mathrm{FP} \cdot \mathrm{FN}}{\sqrt{(\mathrm{TP} + \mathrm{FP}) #' (\mathrm{TP} + \mathrm{FN}) (\mathrm{TN} + \mathrm{FP}) (\mathrm{TN} + \mathrm{FN})}}, #' }{ #' (TP * TN - FP * FN) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)), #' } #' where \eqn{TP}, \eqn{FP}, \eqn{TN}, \eqn{TP} are the number of true positives, false positives, #' true negatives, and false negatives respectively. #' #' In the multi-class case, the Matthews Correlation Coefficient is defined #' for a multi-class confusion matrix \eqn{C} with \eqn{K} classes: \deqn{ #' \frac{c \cdot s - \sum_k^K p_k \cdot t_k}{\sqrt{(s^2 - \sum_k^K p_k^2) \cdot (s^2 - \sum_k^K t_k^2)}}, #' }{ #' (c * s - sum(pk * tk)) / sqrt((s^2 - sum(pk^2)) * (s^2 - sum(tk^2))), #' } #' where #' - \eqn{s = \sum_i^K \sum_j^K C_{ij}}: total number of samples #' - \eqn{c = \sum_k^K C_{kk}}: total number of correctly predicted samples #' - \eqn{t_k = \sum_i^K C_{ik}}: number of predictions for each class \eqn{k} #' - \eqn{p_k = \sum_j^K C_{kj}}: number of true occurrences for each class \eqn{k}. #' #' @templateVar mid mcc #' @template classif_template #' #' @details #' The above formula is undefined if any of the four sums in the denominator is 0 in the binary case, #' and more generally if either \eqn{s^2 - \sum_k^K p_k^2} or \eqn{s^2 - \sum_k^K t_k^2)} is equal to 0. #' The denominator is then set to 1. #' #' When there are more than two classes, the MCC will no longer range between -1 and +1. #' Instead, the minimum value will be between -1 and 0 depending on the true distribution. #' The maximum value is always +1. #' #' @references #' \url{https://en.wikipedia.org/wiki/Phi_coefficient} #' #' `r format_bib("matthews_1975")` #' #' @inheritParams classif_params #' @param positive (`character(1)`) Name of the positive class in case of binary classification. #' #' @template classif_example #' @export mcc = function(truth, response, positive = NULL, sample_weights = NULL, ...) { assert_classif(truth, response = response) mcc_cm(cm(truth, response, positive, sample_weights)) } mcc_cm = function(m, na_value = NaN) { storage.mode(m) = "double" n_tp = sum(diag(m)) n = sum(m) pk = rowSums(m) tk = colSums(m) nomin = n_tp * n - sum(pk * tk) denom = sqrt((n^2 - sum(pk^2)) * (n^2 - sum(tk^2))) if (denom == 0L) nomin else nomin / denom } #' @include measures.R add_measure(mcc, "Matthews Correlation Coefficient", "classif", -1, 1, FALSE) mlr3measures/R/classif_bacc.R0000644000176200001440000000303114661131133015610 0ustar liggesusers#' @title Balanced Accuracy #' #' @details #' The Balanced Accuracy computes the weighted balanced accuracy, suitable for imbalanced data sets. #' It is defined analogously to the definition in [sklearn](https://scikit-learn.org/). #' #' First, all sample weights \eqn{w_i} are normalized per class so that each class has the same influence: #' \deqn{ #' \hat{w}_i = \frac{w_i}{\sum_{j=1}^n w_j \cdot \mathbf{1}(t_j = t_i)}. #' }{ #' w_hat[i] = w[i] / sum((t == t[i]) * w[i]). #' } #' The Balanced Accuracy is then calculated as #' \deqn{ #' \frac{1}{\sum_{i=1}^n \hat{w}_i} \sum_{i=1}^n \hat{w}_i \cdot \mathbf{1}(r_i = t_i). #' }{ #' 1 / sum(w_hat) * sum((r == t) * w_hat). #' } #' This definition is equivalent to [acc()] with class-balanced sample weights. #' #' @references #' `r format_bib("brodersen_2010", "guyon_2015")` #' #' @templateVar mid bacc #' @template classif_template #' #' @inheritParams classif_params #' @template classif_example #' @export bacc = function(truth, response, sample_weights = NULL, ...) { assert_classif(truth, response = response) if (is.null(sample_weights)) { sample_weights = rep(1, length(truth)) } else { assert_numeric(sample_weights, lower = 0, any.missing = FALSE) } label_weights = vapply(split(sample_weights, truth), sum, NA_real_) sample_weights = sample_weights / label_weights[truth] sample_weights[is.na(sample_weights)] = 0 sum((truth == response) * sample_weights) / sum(sample_weights) } #' @include measures.R add_measure(bacc, "Balanced Accuracy", "classif", 0, 1, FALSE) mlr3measures/R/regr_smape.R0000644000176200001440000000157015170376754015366 0ustar liggesusers#' @title Symmetric Mean Absolute Percent Error #' #' @details #' The Symmetric Mean Absolute Percent Error is defined as \deqn{ #' \frac{2}{n} \sum_{i=1}^n \frac{\left| t_i - r_i \right|}{\left| t_i \right| + \left| r_i \right|}. #' }{ #' 2 * mean(abs(t - r) / (abs(t) + abs(r))). #' } #' #' This measure is undefined if any \eqn{|t| + |r|} is equal to \eqn{0}. #' #' @templateVar mid smape #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export smape = function(truth, response, sample_weights = NULL, na_value = NaN, ...) { assert_regr(truth, response = response, na_value = na_value) denom = abs(truth) + abs(response) if (any(denom < TOL)) { return(na_value) } 2 * wmean(.ae(truth, response) / denom, sample_weights) } #' @include measures.R add_measure(smape, "Symmetric Mean Absolute Percent Error", "regr", 0, 2, TRUE) mlr3measures/R/similarity_jaccard.R0000644000176200001440000000215114472156310017057 0ustar liggesusers#' @title Jaccard Similarity Index #' #' @details #' For two sets \eqn{A} and \eqn{B}, the Jaccard Index is defined as #' \deqn{ #' J(A, B) = \frac{|A \cap B|}{|A \cup B|}. #' }{ #' J(A, B) = length(intersect(A, B)) / length(union(A, B)). #' } #' If more than two sets are provided, the mean of all pairwise scores #' is calculated. #' #' @references #' `r format_bib("jaccard_1901", "bommert_2017", "stabm")` #' #' @templateVar mid jaccard #' @template similarity_template #' #' @details #' This measure is undefined if two or more sets are empty. #' #' @inheritParams similarity_params #' #' @export #' @examples #' set.seed(1) #' sets = list( #' sample(letters[1:3], 1), #' sample(letters[1:3], 2) #' ) #' jaccard(sets) jaccard = function(sets, na_value = NaN, ...) { sets = assert_sets(sets) if (sum(lengths(sets) == 0L) >= 2L) { return(na_value) } scores = map_pairwise(sets, function(A, B) { n_inter = length(intersect(A, B)) n_inter / (length(A) + length(B) - n_inter) }) mean(scores) } #' @include measures.R add_measure(jaccard, "Jaccard Similarity Index", "similarity", 0, 1, FALSE) mlr3measures/R/regr_sse.R0000644000176200001440000000156515056012454015042 0ustar liggesusers#' @title Sum of Squared Errors #' #' @details #' The Sum of Squared Errors is defined as \deqn{ #' \sum_{i=1}^n w_i \left( t_i - r_i \right)^2. #' }{ #' sum(w * (t - r)^2). #' } #' where \eqn{w_i} are unnormalized weights for each observation \eqn{x_i}, defaulting to 1. #' #' @templateVar mid sse #' @template regr_template #' #' @param sample_weights (`numeric()`)\cr #' Vector of non-negative and finite sample weights. #' Must have the same length as `truth`. #' Weights for this function are not normalized. #' Defaults to sample weights 1. #' #' @inheritParams regr_params #' @template regr_example #' @export sse = function(truth, response, sample_weights = NULL, ...) { assert_regr(truth, response = response) wsum(.se(truth, response), sample_weights) } #' @include measures.R add_measure(sse, "Sum of Squared Errors", "regr", 0, Inf, TRUE, obs_loss = "se") mlr3measures/R/regr_ktau.R0000644000176200001440000000150415170376754015222 0ustar liggesusers#' @title Kendall's tau #' #' @details #' Kendall's tau is defined as Kendall's rank correlation coefficient between truth and response. #' It is defined as \deqn{ #' \tau = \frac{(\mathrm{number of concordant pairs)} - #' (\mathrm{number of discordant pairs)}}{\mathrm{(number of pairs)}} #' }{ #' t = (number of concordant pairs) - (number of discordant pairs) / (number of pairs)} #' Calls [stats::cor()] with `method` set to `"kendall"`. #' #' @templateVar mid ktau #' @template regr_template #' #' @references #' `r format_bib("rosset_2006")` #' #' @inheritParams regr_params #' @template regr_example #' @export ktau = function(truth, response, ...) { assert_regr(truth, response = response) cor(truth, response, method = "kendall") } #' @include measures.R add_measure(ktau, "Kendall's tau", "regr", -1, 1, FALSE) mlr3measures/R/confusion_matrix.R0000644000176200001440000000416715170376754016636 0ustar liggesusers#' @title Calculate Binary Confusion Matrix #' #' @description #' Calculates the confusion matrix for a binary classification problem #' once and then calculates all binary confusion measures of this package. #' #' @details #' The binary confusion matrix is defined as \deqn{ #' \begin{pmatrix} #' TP & FP \\ #' FN & TN #' \end{pmatrix}. #' }{ #' matrix(TP, FP, FN, TN). #' } #' If `relative = TRUE`, all values are divided by \eqn{n}. #' #' @inheritParams binary_params #' @param relative (`logical(1)`)\cr #' If `TRUE`, the returned confusion matrix contains relative frequencies #' instead of absolute frequencies. #' @return #' List with two elements: #' * `matrix` stores the calculated confusion matrix. #' * `measures` stores the metrics as named numeric vector. #' #' @export #' @examples #' set.seed(123) #' lvls = c("a", "b") #' truth = factor(sample(lvls, 20, replace = TRUE), levels = lvls) #' response = factor(sample(lvls, 20, replace = TRUE), levels = lvls) #' #' confusion_matrix(truth, response, positive = "a") #' confusion_matrix(truth, response, positive = "a", relative = TRUE) #' confusion_matrix(truth, response, positive = "b") confusion_matrix = function(truth, response, positive, na_value = NaN, relative = FALSE) { assert_binary(truth, response = response, positive = positive, na_value = na_value) assert_flag(relative) m = cm(truth, response, positive) ids = c("acc", "ce", "dor", "fbeta", "fdr", "fnr", "fomr", "fpr", "mcc", "npv", "ppv", "tnr", "tpr") measures = vapply( ids, function(id) { do.call(paste0(id, "_cm"), list(m = m, na_value = na_value)) }, FUN.VALUE = NA_real_, USE.NAMES = FALSE ) names(measures) = replace(ids, ids == "fbeta", "f1") if (relative) { m = m / sum(m) } res = list(matrix = m, measures = measures) class(res) = "confusion_matrix" res } #' @export print.confusion_matrix = function(x, ...) { print(x$matrix) str = sprintf("%-4s: % 1.4f", names(x$measures), x$measures) row = head(rep(seq_len(length(str) %/% 4 + 1), each = 4L), length(str)) lapply(split(str, row), function(x) cat(paste(x, collapse = "; "), "\n")) invisible() } mlr3measures/R/regr_rae.R0000644000176200001440000000160615056260067015020 0ustar liggesusers#' @title Relative Absolute Error #' #' @details #' The Relative Absolute Error is defined as \deqn{ #' \frac{\sum_{i=1}^n \left| t_i - r_i \right|}{\sum_{i=1}^n \left| t_i - \bar{t} \right|}, #' }{ #' sum((t - r)^2) / sum((t - mean(t))^2), #' } #' where \eqn{\bar{t} = \sum_{i=1}^n t_i}. #' This measure is undefined for constant \eqn{t}. #' #' Can be interpreted as absolute error of the predictions relative to a naive model predicting the mean. #' #' @templateVar mid rae #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export rae = function(truth, response, na_value = NaN, ...) { .Deprecated(msg = "rae is deprecated") assert_regr(truth, response = response, na_value = na_value) div(sum(.ae(truth, response)), sum(.ae(truth, mean(truth))), na_value) } #' @include measures.R add_measure(rae, "Relative Absolute Error", "regr", 0, Inf, TRUE) mlr3measures/R/binary_tpr.R0000644000176200001440000000224115170123773015376 0ustar liggesusers#' @title True Positive Rate #' #' @details #' The True Positive Rate is defined as \deqn{ #' \frac{\mathrm{TP}}{\mathrm{TP} + \mathrm{FN}}. #' }{ #' TP / (TP + FN). #' } #' This is also know as "recall", "sensitivity", or "probability of detection". #' #' @details #' This measure is undefined if TP + FN = 0. #' #' @templateVar mid tpr #' @template binary_template #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' `r format_bib("goutte_2005")` #' #' @inheritParams binary_params #' @template binary_example #' @export tpr = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) tpr_cm(cm(truth, response, positive, sample_weights), na_value) } tpr_cm = function(m, na_value = NaN) { div(m[1L, 1L], sum(m[, 1L]), na_value) } #' @export #' @rdname tpr recall = tpr #' @export #' @rdname tpr sensitivity = tpr #' @include measures.R add_measure(tpr, "True Positive Rate", "binary", 0, 1, FALSE) add_measure(recall, "Recall", "binary", 0, 1, FALSE) add_measure(sensitivity, "Sensitivity", "binary", 0, 1, FALSE) mlr3measures/R/binary_tn.R0000644000176200001440000000137515170123773015221 0ustar liggesusers#' @title True Negatives #' #' @details #' This measure counts the true negatives, i.e. the number of #' predictions correctly indicating a negative class label. This is sometimes also called a "correct rejection". #' #' @templateVar mid tn #' @template binary_template #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export tn = function(truth, response, positive, sample_weights = NULL, ...) { assert_binary(truth, response = response, positive = positive) tn_cm(cm(truth, response, positive, sample_weights)) } tn_cm = function(m, na_value = NaN) { m[2L, 2L] } #' @include measures.R add_measure(tn, "True Negatives", "binary", 0, Inf, FALSE) mlr3measures/R/assertions.R0000644000176200001440000000336714472156310015426 0ustar liggesusersassert_binary = function(truth, response = NULL, prob = NULL, positive = NULL, na_value = NULL) { assert_factor(truth, min.len = 1L, n.levels = 2L, any.missing = FALSE) if (!is.null(response)) { assert_factor(response, len = length(truth), any.missing = FALSE) assert_set_equal(levels(truth), levels(response), ordered = TRUE) } if (!is.null(prob)) { assert_numeric(prob, len = length(truth), lower = 0, upper = 1, any.missing = FALSE) } if (!is.null(positive)) { assert_string(positive) assert_choice(positive, levels(truth)) } assert_number(na_value, na.ok = TRUE, null.ok = TRUE) } assert_classif = function(truth, response = NULL, prob = NULL, na_value = NULL) { assert_factor(truth, min.len = 1L, min.levels = 2L, any.missing = FALSE) if (!is.null(response)) { assert_factor(response, len = length(truth), any.missing = FALSE) assert_set_equal(levels(truth), levels(response), ordered = TRUE) } if (!is.null(prob)) { assert_matrix(prob, nrows = length(truth), ncols = nlevels(truth), col.names = "named") assert_numeric(prob, lower = 0, upper = 1, any.missing = FALSE) assert_set_equal(colnames(prob), levels(truth)) } assert_number(na_value, na.ok = TRUE, null.ok = TRUE) } assert_regr = function(truth, response, na_value = NULL) { assert_numeric(truth, min.len = 1L, any.missing = FALSE, finite = TRUE) assert_numeric(response, len = length(truth), any.missing = FALSE, finite = TRUE) assert_number(na_value, na.ok = TRUE, null.ok = TRUE) } assert_sets = function(sets) { assert_list(sets, types = c("character", "integer"), min.len = 2L, any.missing = FALSE) if (is.character(sets[[1L]])) { qassertr(sets, "S[1,]") } else { qassertr(sets, "I") } lapply(sets, unique) } mlr3measures/R/binary_npv.R0000644000176200001440000000157315170123773015403 0ustar liggesusers#' @title Negative Predictive Value #' #' @details #' The Negative Predictive Value is defined as \deqn{ #' \frac{\mathrm{TN}}{\mathrm{FN} + \mathrm{TN}}. #' }{ #' TN / (FN + TN). #' } #' #' @templateVar mid npv #' @template binary_template #' #' @details #' This measure is undefined if FN + TN = 0. #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export npv = function(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, response = response, positive = positive, na_value = na_value) npv_cm(cm(truth, response, positive, sample_weights), na_value) } npv_cm = function(m, na_value = NaN) { div(m[2L, 2L], sum(m[2L, ]), na_value) } #' @include measures.R add_measure(npv, "Negative Predictive Value", "binary", 0, 1, FALSE) mlr3measures/R/regr_mae.R0000644000176200001440000000116414661131133015002 0ustar liggesusers#' @title Mean Absolute Error #' #' @details #' The Mean Absolute Error is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i \left| t_i - r_i \right|, #' }{ #' weighted.mean(abs(t - r), w), #' } #' where \eqn{w_i} are normalized sample weights. #' #' @templateVar mid mae #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export mae = function(truth, response, sample_weights = NULL, ...) { assert_regr(truth, response = response) wmean(.ae(truth, response), sample_weights) } #' @include measures.R add_measure(mae, "Mean Absolute Error", "regr", 0, Inf, TRUE, obs_loss = "ae") mlr3measures/R/regr_sae.R0000644000176200001440000000157015056012454015014 0ustar liggesusers#' @title Sum of Absolute Errors #' #' @details #' The Sum of Absolute Errors is defined as \deqn{ #' \sum_{i=1}^n w_i \left| t_i - r_i \right|. #' }{ #' sum(w * abs((t - r))). #' } #' where \eqn{w_i} are unnormalized weights for each observation \eqn{x_i}, defaulting to 1. #' #' @templateVar mid sae #' @template regr_template #' #' @param sample_weights (`numeric()`)\cr #' Vector of non-negative and finite sample weights. #' Must have the same length as `truth`. #' Weights for this function are not normalized. #' Defaults to sample weights 1. #' #' @inheritParams regr_params #' @template regr_example #' @export sae = function(truth, response, sample_weights = NULL, ...) { assert_regr(truth, response = response) sum(.ae(truth, response), sample_weights) } #' @include measures.R add_measure(sae, "Sum of Absolute Errors", "regr", 0, Inf, TRUE, obs_loss = "ae") mlr3measures/R/regr_maxse.R0000644000176200001440000000077014472156310015363 0ustar liggesusers#' @title Max Squared Error #' #' @details #' The Max Squared Error is defined as \deqn{ #' \max \left( t_i - r_i \right)^2. #' }{ #' max((t - r)^2). #' } #' #' @templateVar mid maxse #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export maxse = function(truth, response, ...) { assert_regr(truth, response = response) max(.se(truth, response)) } #' @include measures.R add_measure(maxse, "Max Squared Error", "regr", 0, Inf, TRUE, obs_loss = "se") mlr3measures/R/regr_rrse.R0000644000176200001440000000174515170376754015240 0ustar liggesusers#' @title Root Relative Squared Error #' #' @details #' The Root Relative Squared Error is defined as \deqn{ #' \sqrt{\frac{\sum_{i=1}^n \left( t_i - r_i \right)^2}{\sum_{i=1}^n \left( t_i - \bar{t} \right)^2}}, #' }{ #' sqrt(sum((t - r)^2) / sum((t - mean(t))^2)), #' } #' where \eqn{\bar{t} = \sum_{i=1}^n t_i}. #' #' Can be interpreted as root of the squared error of the predictions relative to a naive model predicting the mean. #' #' This measure is undefined for constant \eqn{t}. #' #' @templateVar mid rrse #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export rrse = function(truth, response, na_value = NaN, ...) { .Deprecated(msg = "rrse is deprecated") assert_regr(truth, response = response, na_value = na_value) v = var(truth) if (v < TOL) { return(na_value) } sqrt(sum(.se(truth, response)) / (v * (length(truth) - 1L))) } #' @include measures.R add_measure(rrse, "Root Relative Squared Error", "regr", 0, Inf, TRUE) mlr3measures/R/binary_fn.R0000644000176200001440000000145715170123773015204 0ustar liggesusers#' @title False Negatives #' #' @details #' This measure counts the false negatives (type 2 error), i.e. the number of #' predictions indicating a negative class label while in fact it is positive. #' This is sometimes also called a "miss" or an "underestimation". #' #' @templateVar mid fn #' @template binary_template #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export fn = function(truth, response, positive, sample_weights = NULL, ...) { assert_binary(truth, response = response, positive = positive) fn_cm(cm(truth, response, positive, sample_weights)) } fn_cm = function(m, na_value = NaN) { m[2L, 1L] } #' @include measures.R add_measure(fn, "False Negatives", "binary", 0, Inf, TRUE) mlr3measures/R/classif_ce.R0000644000176200001440000000137414661131133015317 0ustar liggesusers#' @title Classification Error #' #' @details #' The Classification Error is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i \mathbf{1} \left( t_i \neq r_i \right), #' }{ #' 1 / n * sum(wi * 1(ti != ri)), #' } #' where \eqn{w_i} are normalized weights for each observation \eqn{x_i}. #' #' @templateVar mid ce #' @template classif_template #' #' @inheritParams classif_params #' @template classif_example #' @export ce = function(truth, response, sample_weights = NULL, ...) { assert_classif(truth, response = response) wmean(truth != response, sample_weights) } ce_cm = function(m, na_value = NaN) { s = sum(m) diag(m) = 0 sum(m) / s } #' @include measures.R add_measure(ce, "Classification Error", "classif", 0, 1, TRUE, obs_loss = "zero_one") mlr3measures/R/classif_auc.R0000644000176200001440000001617215170376754015522 0ustar liggesusers#' @title Multiclass AUC Scores #' #' @details #' Multiclass AUC measures. #' #' * *AUNU*: AUC of each class against the rest, using the uniform class #' distribution. Computes the AUC treating a `c`-dimensional classifier #' as `c` two-dimensional 1-vs-rest classifiers, where classes are assumed to have #' uniform distribution, in order to have a measure which is independent #' of class distribution change (Fawcett 2001). #' * *AUNP*: AUC of each class against the rest, using the a-priori class #' distribution. Computes the AUC treating a `c`-dimensional classifier as `c` #' two-dimensional 1-vs-rest classifiers, taking into account the prior probability of #' each class (Fawcett 2001). #' * *AU1U*: AUC of each class against each other, using the uniform class #' distribution. Computes something like the AUC of `c(c - 1)` binary classifiers #' (all possible pairwise combinations). See Hand (2001) for details. #' * *AU1P*: AUC of each class against each other, using the a-priori class #' distribution. Computes something like AUC of `c(c - 1)` binary classifiers #' while considering the a-priori distribution of the classes as suggested #' in Ferri (2009). Note we deviate from the definition in #' Ferri (2009) by a factor of `c`. #' * *MU*: Multiclass AUC as defined in Kleinman and Page (2019). #' This measure is an average of the pairwise AUCs between all classes. #' The measure was tested against the Python implementation by [Ross Kleinman](https://github.com/kleimanr/auc_mu). #' @templateVar mid mauc_aunu #' @template classif_template #' #' @references #' `r format_bib("fawcett_2001", "ferri_2009", "hand_2001", "kleinman_2019")` #' #' @inheritParams classif_params #' @template classif_example #' @export mauc_aunu = function(truth, prob, na_value = NaN, ...) { assert_classif(truth, prob = prob) if (length(unique(truth)) != nlevels(truth)) { warning("Measure is undefined if there isn't at least one sample per class.") return(na_value) } mean(onevrestauc(prob, truth)) } #' @rdname mauc_aunu #' @export mauc_aunp = function(truth, prob, na_value = NaN, ...) { assert_classif(truth, prob = prob) if (nlevels(truth) <= 1L) { # we multiply AUCs of empty classes with 0. The limit expression actually results # in 0 for those, so aunp *is* defined if at least two classes are present. warning("Measure is undefined if there isn't at least one sample for at least two classes.") return(na_value) } sum(vapply(levels(truth), function(lvl) mean(truth == lvl), FUN.VALUE = NA_real_) * onevrestauc(prob, truth)) } #' @rdname mauc_aunu #' @export mauc_au1u = function(truth, prob, na_value = NaN, ...) { assert_classif(truth, prob = prob) if (length(unique(truth)) != nlevels(truth)) { warning("Measure is undefined if there isn't at least one sample per class.") return(na_value) } sum(colAUC(prob, truth)) / (nlevels(truth) * (nlevels(truth) - 1L)) } #' @rdname mauc_aunu #' @export mauc_au1p = function(truth, prob, na_value = NaN, ...) { assert_classif(truth, prob = prob) if (length(unique(truth)) != nlevels(truth)) { warning("Measure is undefined if there isn't at least one sample per class.") return(na_value) } m = colAUC(prob, truth) weights = table(truth) / length(truth) sum(c(m + t(m)) * c(weights)) / (2L * (nlevels(truth) - 1L)) } #' @rdname mauc_aunu #' @export mauc_mu = function(truth, prob, na_value = NaN, ...) { assert_classif(truth, prob = prob) if (length(unique(truth)) != nlevels(truth)) { warning("Measure is undefined if there isn't at least one sample per class.") return(na_value) } n_classes = nlevels(truth) # partition matrix a = matrix(1, n_classes, n_classes) - diag(n_classes) rownames(a) = levels(truth) # iterate over all pairwise combinations of classes pairwise_combinations = combn(levels(truth), 2, simplify = FALSE) aucs = mlr3misc::map_dbl(pairwise_combinations, function(pair) { # subset predictions to instances where the true class is one of the two paired classes class_i = pair[1] preds_i = prob[truth == class_i, , drop = FALSE] n_i = nrow(preds_i) class_j = pair[2] preds_j = prob[truth == class_j, , drop = FALSE] n_j = nrow(preds_j) # calculate pairwise scores temp_preds = rbind(preds_i, preds_j) temp_labels = c(rep(0, n_i), rep(1, n_j)) v = a[class_i, ] - a[class_j, ] scores = temp_preds %*% v # calculate binary auc i = which(temp_labels == 1) n_pos = length(i) n_neg = length(temp_labels) - n_pos r = rank(scores, ties.method = "average") (mean(r[i]) - (as.numeric(n_pos) + 1) / 2) / as.numeric(n_neg) }) sum(aucs * 1 / length(aucs)) } #' @include measures.R add_measure(mauc_aunu, "Average 1 vs. rest multiclass AUC", "classif", 0, 1, FALSE) add_measure(mauc_aunp, "Weighted average 1 vs. rest multiclass AUC", "classif", 0, 1, FALSE) add_measure(mauc_au1u, "Average 1 vs. 1 multiclass AUC", "classif", 0, 1, FALSE) add_measure(mauc_au1p, "Weighted average 1 vs. 1 multiclass AUC", "classif", 0, 1, FALSE) add_measure(mauc_mu, "Multiclass mu AUC", "classif", 0, 1, FALSE) # returns a numeric length nlevel(truth), with one-vs-rest AUC onevrestauc = function(prob, truth) { ntotal = nrow(prob) vapply( levels(truth), function(cls) { nrest = sum(truth != cls) if (nrest == ntotal) { # this class has no members. What happened? # - If we were called by mauc_aunu --> this does never happen, because we return(NA) if there are empty classes # - If we were called by mauc_aunp --> this value gets multiplied with 0 and should result in 0 # therefore we don't want to return Inf here, but a final value that does not matter in the end return(0) } r = rank(c(prob[truth == cls, cls], prob[truth != cls, cls]), ties.method = "average") # simplify the following: # (sum(r[seq_len(nrest)]) - nrest * (nrest + 1) / 2) / (nrest * (ntotal - nrest)) (mean(r[seq_len(ntotal - nrest)]) - (ntotal - nrest + 1) / 2) / nrest }, FUN.VALUE = NA_real_ ) } # calculates \cite{hand_2001} pairwise asymmetric(!) AUC matrix # nolint next colAUC = function(prob, truth) { prob = as.matrix(prob) # turn numeric vector to column if necessary truth = as.factor(truth) # turn logical to factor # conditional_auc[i, j] is A(i | j) as defined in \cite{hand_2001}: # "the probability that a randomly drawn member of class j will have a # lower estimated probability of belonging to class i than a randomly # drawn member of class i" conditional_auc = matrix(0, nlevels(truth), nlevels(truth)) rownames(conditional_auc) = levels(truth) colnames(conditional_auc) = levels(truth) for (i in levels(truth)) { ni = sum(truth == i) # avoid integer overflow for (j in levels(truth)) { if (i == j) { next } nj = sum(truth == j) # avoid integer overflow r = rank(c(prob[truth == i, i], prob[truth == j, i]), ties.method = "average") # simplify the following: # conditional_auc[i, j] = (sum(r[seq_len(nj)]) - nj * (nj + 1) / 2) / (ni * nj) conditional_auc[i, j] = (mean(r[seq_len(ni)]) - (ni + 1) / 2) / nj } } conditional_auc } mlr3measures/R/binary_auc.R0000644000176200001440000001237615170370742015353 0ustar liggesusers#' @title Area Under the ROC Curve #' #' @details #' Computes the area under the Receiver Operator Characteristic (ROC) curve. #' The AUC can be interpreted as the probability that a randomly chosen positive observation #' has a higher predicted probability than a randomly chosen negative observation. #' #' For \eqn{n^+} positive and \eqn{n^-} negative observations with \eqn{R_i^+} the rank of the #' \eqn{i}-th positive observation's predicted probability (average ranks for ties), the AUC #' is estimated as \deqn{ #' \widehat{\operatorname{AUC}} = \frac{1}{n^+ n^-} \left( \sum_{i=1}^{n^+} R_i^+ \; - \; \frac{n^+(n^+ + 1)}{2} \right). #' }{ #' AUC = (sum(R_pos) - n_pos * (n_pos + 1) / 2) / (n_pos * n_neg). #' } #' #' If `sample_weights` are provided, let \eqn{w_i^+} be the weight of the \eqn{i}-th positive observation with #' predicted probability \eqn{p_i^+}, \eqn{W^+ = \sum_i w_i^+}, and \eqn{W^-} the total weight of the #' negative observations. Define the weighted Mann-Whitney contribution of positive observation \eqn{i} as #' \eqn{U_i^w = W_{< p_i^+}^- + \tfrac{1}{2} W_{= p_i^+}^-}, #' i.e. the total weight of negative observations with a smaller predicted probability plus half the weight #' of negatives tied with \eqn{p_i^+}. The weighted AUC is then \deqn{ #' \widehat{\operatorname{AUC}}_w = \frac{1}{W^+ W^-} \sum_{i=1}^{n^+} w_i^+ U_i^w. #' }{ #' AUC_w = sum(w_pos * U_pos) / (W_pos * W_neg). #' } #' #' @templateVar mid auc #' @template binary_template #' #' @details #' This measure is undefined if the true values are either all positive or #' all negative. #' #' @inheritParams binary_params #' #' @references #' `r format_bib("youden_1950")` #' #' @export #' @examples #' truth = factor(c("a", "a", "a", "b")) #' prob = c(.6, .7, .1, .4) #' auc(truth, prob, "a") auc = function(truth, prob, positive, sample_weights = NULL, na_value = NaN, ...) { assert_binary(truth, prob = prob, positive = positive, na_value = na_value) i = which(truth == positive) # Unweighted path uses the rank-sum identity via a single C-level rank() call. # Weighted path computes the Mann-Whitney U directly: a sort + cumsum over # negative weights, with rle() to share values across tied prob groups. if (is.null(sample_weights)) { # Unweighted: AUC = (sum_{i in pos} r_i - n+(n+ + 1)/2) / (n+ * n-), # where r_i is the average rank of prob[i] in the combined sample. n_pos = length(i) n_neg = length(truth) - n_pos if (n_pos == 0L || n_neg == 0L) { return(na_value) } r = rank(prob, ties.method = "average") # simplifying the following: # (sum(r[i]) - n_pos * (n_pos + 1L) / 2L) / (n_pos * n_neg) (mean(r[i]) - (as.numeric(n_pos) + 1) / 2) / as.numeric(n_neg) } else { assert_numeric(sample_weights, lower = 0, finite = TRUE, any.missing = FALSE, len = length(truth)) # Weighted AUC via the direct Mann-Whitney form: # AUC_w = sum_i w_i^+ U_i^w / (W+ W-), # with U_i^w = W_ 0}. #' #' @templateVar mid linex #' @template regr_template #' @param a (`numeric(1)`)\cr #' Shape parameter controlling asymmetry. #' Negative values penalize overestimation more, positive values penalize underestimation more. #' As `a` approaches 0, the loss resembles squared error loss. Default is `-1`. #' @param b (`numeric(1)`)\cr #' Positive scaling factor for the loss. Larger values increase the loss magnitude. #' Default is `1`. #' #' @references #' `r format_bib("varian_1975")` #' #' @inheritParams regr_params #' @template regr_example #' @export linex = function(truth, response, a = -1, b = 1, ...) { assert_regr(truth, response = response) assert_number(a) assert_number(b, lower = 0) if (a == 0) { stop("'a' can't be 0") } e = truth - response we = a * e b * (exp(we) - we - 1) } #' @include measures.R add_measure(linex, "Linear-exponential Loss (per observation)", "regr", 0, Inf, TRUE, aggregated = FALSE) mlr3measures/R/binary_fp.R0000644000176200001440000000143615170123773015203 0ustar liggesusers#' @title False Positives #' #' @details #' This measure counts the false positives (type 1 error), i.e. the number of #' predictions indicating a positive class label while in fact it is negative. #' This is sometimes also called a "false alarm". #' #' @templateVar mid fp #' @template binary_template #' #' @references #' \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} #' #' @inheritParams binary_params #' @template binary_example #' @export fp = function(truth, response, positive, sample_weights = NULL, ...) { assert_binary(truth, response = response, positive = positive) fp_cm(cm(truth, response, positive, sample_weights)) } fp_cm = function(m, na_value = NaN) { m[1L, 2L] } #' @include measures.R add_measure(fp, "False Positives", "binary", 0, Inf, TRUE) mlr3measures/R/regr_medse.R0000644000176200001440000000104714661131133015335 0ustar liggesusers#' @title Median Squared Error #' #' @details #' The Median Squared Error is defined as \deqn{ #' \mathop{\mathrm{median}} \left[ \left( t_i - r_i \right)^2 \right]. #' }{ #' median((t - r)^2). #' } #' #' @templateVar mid medse #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export medse = function(truth, response, ...) { assert_regr(truth, response = response) median(.se(truth, response)) } #' @include measures.R add_measure(medse, "Median Squared Error", "regr", 0, Inf, TRUE, obs_loss = "se") mlr3measures/R/regr_bias.R0000644000176200001440000000112715056012322015152 0ustar liggesusers#' @title Bias #' #' @details #' The Bias is defined as \deqn{ #' \frac{1}{n} \sum_{i=1}^n w_i \left( r_i - t_i \right), #' }{ #' weighted.mean(r - t, w), #' } #' where \eqn{w_i} are normalized sample weights. #' Good predictions score close to 0. #' #' @templateVar mid bias #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export bias = function(truth, response, sample_weights = NULL, ...) { assert_regr(truth, response = response) wmean(response - truth, sample_weights) } #' @include measures.R add_measure(bias, "Bias", "regr", -Inf, Inf, NA) mlr3measures/R/regr_pinball.R0000644000176200001440000000162714661131133015665 0ustar liggesusers#' @title Average Pinball Loss #' #' @details #' The pinball loss for quantile regression is defined as \deqn{ #' \text{Average Pinball Loss} = \frac{1}{n} \sum_{i=1}^{n} w_{i} #' \begin{cases} #' q \cdot (t_i - r_i) & \text{if } t_i \geq r_i \\ #'(1 - q) \cdot (r_i - t_i) & \text{if } t_i < r_i #' \end{cases} #' } #' where \eqn{q} is the quantile and \eqn{w_i} are normalized sample weights. #' #' #' @templateVar mid pinball #' @template regr_template #' #' @inheritParams regr_params #' #' @param alpha `numeric(1)`\cr #' The quantile to compute the pinball loss. #' #' @template regr_example #' @export pinball = function(truth, response, sample_weights = NULL, alpha = 0.5, ...) { assert_regr(truth, response = response) diff = truth - response wmean(ifelse(diff >= 0, alpha * diff, (1 - alpha) * -diff), sample_weights) } #' @include measures.R add_measure(pinball, "Pinball", "regr", -Inf, Inf, TRUE) mlr3measures/R/regr_se.R0000644000176200001440000000103714661131133014646 0ustar liggesusers#' @title Squared Error (per observation) #' #' @details #' Calculates the per-observation squared error as \deqn{ #' \left( t_i - r_i \right)^2. #' }{ #' (t - r)^2. #' } #' #' @templateVar mid se #' @template regr_template #' @inheritParams regr_params #' #' @export se = function(truth, response, ...) { assert_regr(truth, response) .se(truth, response) } .se = function(truth, response) { (truth - response)^2 } #' @include measures.R add_measure(se, "Squared Error (per observation)", "regr", 0, Inf, TRUE, aggregated = FALSE) mlr3measures/R/binary_prauc.R0000644000176200001440000000235714661131133015704 0ustar liggesusers#' @title Area Under the Precision-Recall Curve #' #' @details #' Computes the area under the Precision-Recall curve (PRC). #' The PRC can be interpreted as the relationship between precision and recall (sensitivity), #' and is considered to be a more appropriate measure for unbalanced datasets than the ROC curve. #' The AUC-PRC is computed by integration of the piecewise function. #' #' @templateVar mid prauc #' @template binary_template #' #' @details #' This measure is undefined if the true values are either all positive or #' all negative. #' #' @inheritParams binary_params #' #' @references #' `r format_bib("davis_2006")` #' #' @export #' @examples #' truth = factor(c("a", "a", "a", "b")) #' prob = c(.6, .7, .1, .4) #' prauc(truth, prob, "a") prauc = function(truth, prob, positive, na_value = NaN, ...) { assert_binary(truth, prob = prob, positive = positive, na_value = na_value) i = which(truth == positive) n_pos = length(i) n_neg = length(truth) - n_pos if (n_pos == 0L || n_neg == 0L) { return(na_value) } truth = ifelse(truth == positive, 1, 0) PRROC::pr.curve( scores.class0 = prob, weights.class0 = truth )[[2]] } #' @include measures.R add_measure(prauc, "Precision-Recall Curve", "binary", 0, 1, FALSE) mlr3measures/R/classif_zero_one.R0000644000176200001440000000165314661131133016550 0ustar liggesusers#' @title Zero-One Classification Loss (per observation) #' #' @description #' Calculates the per-observation 0/1 (zero-one) loss as \deqn{ #' \mathbf{1} (t_i \neq r_1). #' }{ #' 1(t != r). #' } #' The 1/0 (one-zero) loss is equal to 1 - zero-one and calculated as \deqn{ #' \mathbf{1} (t_i = r_i). #' }{ #' 1(t_i = r_i). #' } #' #' @templateVar mid zero_one #' @template classif_template #' @inheritParams classif_params #' #' @export zero_one = function(truth, response, ...) { assert_classif(truth, response) as.integer(truth != response) } #' @rdname zero_one #' @export one_zero = function(truth, response, ...) { assert_classif(truth, response) as.integer(truth == response) } #' @include measures.R add_measure(zero_one, "Zero-One Classification Loss", "classif", 0, 1, TRUE, aggregated = FALSE) #' @include measures.R add_measure(one_zero, "One-Zero Utility Function", "classif", 0, 1, FALSE, aggregated = FALSE) mlr3measures/R/regr_ae.R0000644000176200001440000000104214661131133014620 0ustar liggesusers#' @title Absolute Error (per observation) #' #' @details #' Calculates the per-observation absolute error as \deqn{ #' \left| t_i - r_i \right|. #' }{ #' abs(t - r). #' } #' #' @templateVar mid ae #' @template regr_template #' @inheritParams regr_params #' #' @export ae = function(truth, response, ...) { assert_regr(truth, response) .ae(truth, response) } .ae = function(truth, response) { abs(truth - response) } #' @include measures.R add_measure(ae, "Absolute Error (per observation)", "regr", 0, Inf, TRUE, aggregated = FALSE) mlr3measures/R/classif_logloss.R0000644000176200001440000000340115170376754016423 0ustar liggesusers#' @title Log Loss #' #' @details #' The Log Loss (a.k.a Bernoulli Loss, Logistic Loss, Cross-Entropy Loss) is defined as #' \deqn{ #' -\frac{1}{n} \sum_{i=1}^n w_i \log \left( p_i \right ) #' }{ #' -weighted.mean(log(p), w) #' } #' where \eqn{p_i}{p} is the probability for the true class of observation \eqn{i}, #' and \eqn{w_i} are normalized weights for each observation \eqn{x_i}. #' #' @templateVar mid logloss #' @template classif_template #' #' @inheritParams classif_params #' @param eps (`numeric(1)`)\cr #' Probabilities are clipped to `max(eps, min(1 - eps, p))`. #' Otherwise the measure would be undefined for probabilities `p = 0` and `p = 1`. #' @export #' @examples #' set.seed(1) #' lvls = c("a", "b", "c") #' truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) #' prob = matrix(runif(3 * 10), ncol = 3, dimnames = list(NULL, lvls)) #' prob = t(apply(prob, 1, function(x) x / sum(x))) #' logloss(truth, prob) logloss = function(truth, prob, sample_weights = NULL, eps = 1e-15, ...) { assert_classif(truth, prob = prob) assert_number(eps, lower = 0, upper = 1) ii = match(as.character(truth), colnames(prob)) p = prob[cbind(seq_len(nrow(prob)), ii)] p = pmax(eps, pmin(1 - eps, p)) -wmean(log(p), sample_weights) } #' @include measures.R add_measure(logloss, "Log Loss", "classif", 0, Inf, TRUE, obs_loss = "obs_logloss") #' @title Observation-wise Log Loss #' #' @description #' Observation-wise loss function for [logloss()]. #' #' @export #' @keywords internal obs_logloss = function(truth, prob, eps = 1e-15, ...) { assert_classif(truth, prob = prob) assert_number(eps, lower = 0, upper = 1) ii = match(as.character(truth), colnames(prob)) p = prob[cbind(seq_len(nrow(prob)), ii)] p = pmax(eps, pmin(1 - eps, p)) -log(p) } mlr3measures/R/regr_maxae.R0000644000176200001440000000101114472156310015326 0ustar liggesusers#' @title Max Absolute Error #' #' @details #' The Max Absolute Error is defined as \deqn{ #' \max \left( \left| t_i - r_i \right| \right). #' }{ #' max(abs(t - r)). #' } #' #' @templateVar mid maxae #' @template regr_template #' #' @inheritParams regr_params #' @template regr_example #' @export maxae = function(truth, response, ...) { assert_regr(truth, response = response) max(.ae(truth, response)) } #' @include measures.R add_measure(maxae, "Max Absolute Error", "regr", 0, Inf, TRUE, obs_loss = "ae") mlr3measures/R/bibentries.R0000644000176200001440000002242614661143121015353 0ustar liggesusersformat_bib = function(...) { str = vapply(list(...), function(entry) tools::toRd(bibentries[[entry]]), FUN.VALUE = "") paste0(str, collapse = "\n\n") } #' @importFrom utils bibentry bibentries = c( # nolint start brodersen_2010 = bibentry("inproceedings", title = "The Balanced Accuracy and Its Posterior Distribution", author = "Kay Henning Brodersen and Cheng Soon Ong and Klaas Enno Stephan and Joachim M. Buhmann", year = "2010", booktitle = "2010 20th International Conference on Pattern Recognition", publisher = "IEEE", doi = "10.1109/icpr.2010.764" ), davis_2006 = bibentry("inproceedings", title = "The relationship between precision-recall and ROC curves", author = "Jesse Davis and Mark Goadrich", year = "2006", booktitle = "Proceedings of the 23rd International Conference on Machine Learning", publisher = "ACM", isbn = "9781595933836" ), de_myttenaere_2016 = bibentry("article", title = "Mean Absolute Percentage Error for regression models", author = "de Myttenaere, Arnaud and Golden, Boris and Le Grand, B\u00e9n\u00e9dicte and Rossi, Fabrice", year = "2016", journal = "Neurocomputing", publisher = "Elsevier BV", volume = "192", pages = "38-48", doi = "10.1016/j.neucom.2015.12.114", issn = "0925-2312" ), guyon_2015 = bibentry("inproceedings", title = "Design of the 2015 {ChaLearn} {AutoML} challenge", author = "Isabelle Guyon and Kristin Bennett and Gavin Cawley and Hugo Jair Escalante and Sergio Escalera and Tin Kam Ho and Nuria Macia and Bisakha Ray and Mehreen Saeed and Alexander Statnikov and Evelyne Viegas", year = "2015", booktitle = "2015 International Joint Conference on Neural Networks ({IJCNN})", publisher = "IEEE", doi = "10.1109/ijcnn.2015.7280767" ), matthews_1975 = bibentry("article", title = "Comparison of the predicted and observed secondary structure of T4 phage lysozyme", author = "Brian W. Matthews", year = "1975", journal = "Biochimica et Biophysica Acta ({BBA}) - Protein Structure", publisher = "Elsevier {BV}", volume = "405", number = "2", pages = "442--451", doi = "10.1016/0005-2795(75)90109-9" ), rijsbergen_1979 = bibentry("book", title = "Information Retrieval", author = "Rijsbergen, C. J. Van", year = "1979", publisher = "Butterworth-Heinemann", address = "Newton, MA, USA", isbn = "408709294", edition = "2nd" ), rosset_2006 = bibentry("article", title = "Ranking-based evaluation of regression models", author = "Saharon Rosset and Claudia Perlich and Bianca Zadrozny", year = "2006", journal = "Knowledge and Information Systems", publisher = "Springer Science and Business Media {LLC}", volume = "12", number = "3", pages = "331--353", doi = "10.1007/s10115-006-0037-3" ), goutte_2005 = bibentry("inproceedings", doi = "10.1007/978-3-540-31865-1_25", year = "2005", publisher = "Springer Berlin Heidelberg", pages = "345--359", author = "Cyril Goutte and Eric Gaussier", title = "A Probabilistic Interpretation of Precision, Recall and F-Score, with Implication for Evaluation", booktitle = "Lecture Notes in Computer Science" ), youden_1950 = bibentry("article", title = "Index for rating diagnostic tests", author = "W. J. Youden", year = "1950", journal = "Cancer", publisher = "Wiley", volume = "3", number = "1", pages = "32--35", doi = "10.1002/1097-0142(1950)3:1<32::aid-cncr2820030106>3.0.co;2-3" ), brier_1950 = bibentry("article", doi = "10.1175/1520-0493(1950)078<0001:vofeit>2.0.co;2", year = "1950", month = "jan", publisher = "American Meteorological Society", volume = "78", number = "1", pages = "1--3", author = "Glenn W. Brier", title = "Verification of forecasts expressed in terms of probability", journal = "Monthly Weather Review" ), ferri_2009 = bibentry("article", doi = "10.1016/j.patrec.2008.08.010", year = "2009", month = "jan", publisher = "Elsevier", volume = "30", number = "1", pages = "27--38", author = "Ferri, C\u00e9sar and Hern\u00e1ndez-Orallo, Jos\u00e9 and Modroiu, R", title = "An experimental comparison of performance measures for classification", journal = "Pattern Recognition Letters" ), hand_2001 = bibentry("article", title = "A simple generalisation of the area under the ROC curve for multiple class classification problems", author = "Hand, David J and Till, Robert J", journal = "Machine learning", volume = "45", number = "2", pages = "171--186", year = "2001", publisher = "Springer" ), fawcett_2001 = bibentry("inproceedings", title = "Using rule sets to maximize ROC performance", author = "Fawcett, Tom", booktitle = "Proceedings 2001 IEEE international conference on data mining", pages = "131--138", year = "2001", organization = "IEEE" ), jaccard_1901 = bibentry("article", doi = "10.5169/SEALS-266450", author = "Jaccard, Paul", title = "\u00c9tude comparative de la distribution florale dans une portion des Alpes et du Jura", journal = "Bulletin de la Soci\u00e9t\u00e9 Vaudoise des Sciences Naturelles", volume = "37", pages = "547-579", publisher = "Imprimerie Corbaz & Comp.", year = "1901" ), bommert_2017 = bibentry("article", doi = "10.1155/2017/7907163", year = "2017", publisher = "Hindawi Limited", volume = "2017", pages = "1--18", author = "Andrea Bommert and J\u00f6rg Rahnenf\u00fchrer and Michel Lang", title = "A Multicriteria Approach to Find Predictive and Sparse Models with Stable Feature Selection for High-Dimensional Data", journal = "Computational and Mathematical Methods in Medicine" ), nogueira_2016 = bibentry("incollection", doi = "10.1007/978-3-319-46227-1_28", year = "2016", publisher = "Springer International Publishing", pages = "442--457", author = "Sarah Nogueira and Gavin Brown", title = "Measuring the Stability of Feature Selection", booktitle = "Machine Learning and Knowledge Discovery in Databases" ), stabm = bibentry("article", title = "{stabm}: Stability Measures for Feature Selection", author = "Andrea Bommert and Michel Lang", journal = "Journal of Open Source Software", year = "2021", doi = "10.21105/joss.03010", publisher = "The Open Journal", volume = "6", number = "59", pages = "3010" ), he_2009 = bibentry("article", title = "Learning from Imbalanced Data", author = "Haibo He and Edwardo A. Garcia", journal = "IEEE Transactions on knowledge and data engineering", volume = "21", number = "9", pages = "1263--1284", year = "2009", doi = "10.1109/TKDE.2008.239" ), kleinman_2019 = bibentry("incollection", title = "AUC mu: A Performance Metric for Multi-Class Machine Learning Models", author = "Ross Kleiman and David Page", booktitle = "Proceedings of the 36th International Conference on Machine Learning", pages = "3439--3447", year = "2019", editor = "Chaudhuri, Kamalika and Salakhutdinov, Ruslan", volume = "97", series = "Proceedings of Machine Learning Research", publisher = "PMLR" ), varian_1975 = bibentry("incollection", title = "A Bayesian Approach to Real Estate Assessment", author = "Varian, Hal R.", booktitle = "Studies in Bayesian Econometrics and Statistics: In Honor of Leonard J. Savage", editor = "Stephen E. Fienberg and Arnold Zellner", year = "1975", publisher = "North-Holland", address = "Amsterdam", pages = "195--208" ) ) # nolint end mlr3measures/NAMESPACE0000644000176200001440000000253515170370742014126 0ustar liggesusers# Generated by roxygen2: do not edit by hand S3method(as.data.frame,MeasureEnv) S3method(print,MeasureEnv) S3method(print,confusion_matrix) export(acc) export(ae) export(ape) export(auc) export(bacc) export(bbrier) export(bias) export(ce) export(confusion_matrix) export(dor) export(fbeta) export(fdr) export(fn) export(fnr) export(fomr) export(fp) export(fpr) export(gmean) export(gpr) export(jaccard) export(ktau) export(linex) export(logloss) export(mae) export(mape) export(mauc_au1p) export(mauc_au1u) export(mauc_aunp) export(mauc_aunu) export(mauc_mu) export(maxae) export(maxse) export(mbrier) export(mcc) export(measures) export(medae) export(medse) export(mse) export(msle) export(npv) export(obs_logloss) export(one_zero) export(pbias) export(phi) export(pinball) export(ppv) export(prauc) export(precision) export(rae) export(recall) export(rmse) export(rmsle) export(rrse) export(rse) export(rsq) export(sae) export(se) export(se_binary) export(sensitivity) export(sle) export(smape) export(specificity) export(srho) export(sse) export(tn) export(tnr) export(tp) export(tpr) export(zero_one) import(checkmate) importFrom(stats,ave) importFrom(stats,contr.treatment) importFrom(stats,cor) importFrom(stats,median) importFrom(stats,relevel) importFrom(stats,var) importFrom(stats,xtabs) importFrom(utils,bibentry) importFrom(utils,combn) importFrom(utils,head) mlr3measures/NEWS.md0000644000176200001440000000420515170426306013777 0ustar liggesusers# mlr3measures 1.3.0 * feat: `auc()` gains `sample_weights` for weighted AUC. * feat: Confusion matrix based measures gain `sample_weights` (#71). * feat: `mbrier()` and `smape()` gain `sample_weights` (#72). # mlr3measures 1.2.0 * feat: Add observation-wise loss functions for `bbrier` and `logloss`. # mlr3measures 1.1.0 * fix: Define bias measures correctly. * BREAKING CHANGE: Deprecate `rse`, `rsq`, `rrse`, and `rae`. * feat: Measures `sae` and `sse` gain `sample_weights` (unnormalized weights). # mlr3measures 1.0.0 * Added new measure `linex` (Linear-Exponential Loss). * Added new measure `pinball` (Average Pinball Loss). * Added new measure `mauc_mu` (Mu AUC). # mlr3measures 0.6.0 * Added binary classification measures `gmean` and `gpr`. * Added new measure `mcc` (Matthews Correlation Coefficient). # mlr3measures 0.5.0 * Added some observation-wise loss functions: `ae`, `ape`, `se`, `sle`, and `zero_one`, # mlr3measures 0.4.1 * Calculation of `fbeta` is now numerically more stable. * Improved documentation. # mlr3measures 0.4.0 * New measures to calculate set similarities: `jaccard` and `phi`. * Many measures now support sample weights. Supported measures have `$sample_weights` set to `TRUE` in the `measures` environment. # mlr3measures 0.3.1 * Fix edge case for `msle` and `rmsle`. # mlr3measures 0.3.0 * New measure `prauc` (area under precision-recall curve) (#22) implemented in package `PRROC`. * Removed dependency on orphaned package `bibtex`. # mlr3measures 0.2.0 * New measures: `mauc_au1p`, `mauc_au1u`, `mauc_aunp`, and `mauc_aunu`. * Fixed noLD check warnings. # mlr3measures 0.1.3 * Added a `packageStartupMessage()` advising against attaching this package. * Added new measure `bbrier` (binary Brier Score). * Added new measure `mbrier` (multi-class Brier Score). * Improved numerical stability for measure `mcc`. # mlr3measures 0.1.2 * Fixed integer overflows in `auc`, `dor` and `mcc`. # mlr3measures 0.1.1 * Added new measure `bacc` (Balanced Accuracy). * Fixed some tests which stochastically failed. * The name of the measure is now also stored in the meta data. # mlr3measures 0.1.0 Initial release. mlr3measures/README.md0000644000176200001440000000306614472156310014163 0ustar liggesusers# mlr3measures Package website: [release](https://mlr3measures.mlr-org.com/) | [dev](https://mlr3measures.mlr-org.com/dev/) [![r-cmd-check](https://github.com/mlr-org/mlr3measures/actions/workflows/r-cmd-check.yml/badge.svg)](https://github.com/mlr-org/mlr3measures/actions/workflows/r-cmd-check.yml) [![CRAN Status](https://www.r-pkg.org/badges/version-ago/mlr3measures)](https://cran.r-project.org/package=mlr3measures) [![StackOverflow](https://img.shields.io/badge/stackoverflow-mlr3-orange.svg)](https://stackoverflow.com/questions/tagged/mlr3) [![Mattermost](https://img.shields.io/badge/chat-mattermost-orange.svg)](https://lmmisld-lmu-stats-slds.srv.mwn.de/mlr_invite/) Implements multiple performance measures for supervised learning. Includes over 40 measures for regression and classification. Additionally, meta information about the performance measures can be queried, e.g. what the best and worst possible performances scores are. Internally, [checkmate](https://CRAN.R-project.org/package=checkmate) is used to check arguments efficiently - no other runtime dependencies. The [function reference](https://mlr3measures.mlr-org.com/reference/index.html) gives an encompassing overview over implemented measures. Note that explicitly loading this package is not required if you want to use any of these measures in [mlr3](https://mlr3.mlr-org.com). Also note that we advise against attaching the package via `library()` to avoid namespace clashes. Instead, load the namespace via `requireNamespace()` and use the `::` operator. mlr3measures/build/0000755000176200001440000000000015170426652014003 5ustar liggesusersmlr3measures/build/partial.rdb0000644000176200001440000000007515170426652016132 0ustar liggesusers‹‹àb```b`aef`b1…À€… H02°0piÖ¼ÄÜÔb C"Éð+ƒ[7mlr3measures/man/0000755000176200001440000000000015170376754013466 5ustar liggesusersmlr3measures/man/mse.Rd0000644000176200001440000000374115056260067014536 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_mse.R \name{mse} \alias{mse} \title{Mean Squared Error} \usage{ mse(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Mean Squared Error is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i \left( t_i - r_i \right)^2, }{ weighted.mean((t - r)^2, w), } where \eqn{w_i} are normalized sample weights. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) mse(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/classif_params.Rd0000644000176200001440000000215514472156310016733 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/roxygen.R \name{classif_params} \alias{classif_params} \title{Classification Parameters} \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the same levels and length as \code{truth}.} \item{prob}{(\code{matrix()})\cr Matrix of predicted probabilities, each column is a vector of probabilities for a specific class label. Columns must be named with levels of \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \description{ Classification Parameters } \keyword{internal} mlr3measures/man/se_binary.Rd0000644000176200001440000000046515111057476015725 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_bbrier.R \name{se_binary} \alias{se_binary} \title{Binary Squared Error} \usage{ se_binary(truth, prob, positive, ...) } \description{ Observation-wise loss function for \code{\link[=bbrier]{bbrier()}}. } \keyword{internal} mlr3measures/man/medae.Rd0000644000176200001440000000326515056260067015026 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_medae.R \name{medae} \alias{medae} \title{Median Absolute Error} \usage{ medae(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Median Absolute Error is defined as \deqn{ \mathop{\mathrm{median}} \left| t_i - r_i \right|. }{ median(abs(t - r)). } } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) medae(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/bias.Rd0000644000176200001440000000375515056260067014675 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_bias.R \name{bias} \alias{bias} \title{Bias} \usage{ bias(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Bias is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i \left( r_i - t_i \right), }{ weighted.mean(r - t, w), } where \eqn{w_i} are normalized sample weights. Good predictions score close to 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{(-\infty, \infty)}{(-Inf, Inf)} \item Minimize: \code{NA} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) bias(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/gpr.Rd0000644000176200001440000000510415170123773014534 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_gpr.R \name{gpr} \alias{gpr} \title{Geometric Mean of Precision and Recall} \usage{ gpr(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ Calculates the geometric mean of \code{\link[=precision]{precision()}} P and \code{\link[=recall]{recall()}} R as \deqn{ \sqrt{\mathrm{P} \cdot \mathrm{R}}. }{ sqrt(P * R) } This measure is undefined if precision or recall is undefined, i.e. if TP + FP = 0 or if TP + FN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) gpr(truth, response, positive = "a") } \references{ He H, Garcia EA (2009). \dQuote{Learning from Imbalanced Data.} \emph{IEEE Transactions on knowledge and data engineering}, \bold{21}(9), 1263--1284. \doi{10.1109/TKDE.2008.239}. } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/mlr3measures-package.Rd0000644000176200001440000000212315056012322017743 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/zzz.R \docType{package} \name{mlr3measures-package} \alias{mlr3measures} \alias{mlr3measures-package} \title{mlr3measures: Performance Measures for 'mlr3'} \description{ Implements multiple performance measures for supervised learning. Includes over 40 measures for regression and classification. Additionally, meta information about the performance measures can be queried, e.g. what the best and worst possible performances scores are. } \seealso{ Useful links: \itemize{ \item \url{https://mlr3measures.mlr-org.com} \item \url{https://github.com/mlr-org/mlr3measures} \item Report bugs at \url{https://github.com/mlr-org/mlr3measures/issues} } } \author{ \strong{Maintainer}: Marc Becker \email{marcbecker@posteo.de} (\href{https://orcid.org/0000-0002-8115-0400}{ORCID}) Authors: \itemize{ \item Michel Lang \email{michellang@gmail.com} (\href{https://orcid.org/0000-0001-9754-0393}{ORCID}) \item Lona Koers } Other contributors: \itemize{ \item Martin Binder \email{mlr.developer@mb706.com} [contributor] } } mlr3measures/man/phi.Rd0000644000176200001440000000453714472156310014532 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/similarity_phi.R \name{phi} \alias{phi} \title{Phi Coefficient Similarity} \usage{ phi(sets, p, na_value = NaN, ...) } \arguments{ \item{sets}{(\code{list()})\cr List of character or integer vectors. \code{sets} must have at least 2 elements.} \item{p}{(\code{integer(1)})\cr Total number of possible elements.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare two or more sets w.r.t. their similarity. } \details{ The Phi Coefficient is defined as the Pearson correlation between the binary representation of two sets \eqn{A} and \eqn{B}. The binary representation for \eqn{A} is a logical vector of length \eqn{p} with the i-th element being 1 if the corresponding element is in \eqn{A}, and 0 otherwise. If more than two sets are provided, the mean of all pairwise scores is calculated. This measure is undefined if one set contains none or all possible elements. } \section{Meta Information}{ \itemize{ \item Type: \code{"similarity"} \item Range: \eqn{[-1, 1]}{[-1, 1]} \item Minimize: \code{FALSE} } } \examples{ set.seed(1) sets = list( sample(letters[1:3], 1), sample(letters[1:3], 2) ) phi(sets, p = 3) } \references{ Nogueira S, Brown G (2016). \dQuote{Measuring the Stability of Feature Selection.} In \emph{Machine Learning and Knowledge Discovery in Databases}, 442--457. Springer International Publishing. \doi{10.1007/978-3-319-46227-1_28}. Bommert A, Rahnenführer J, Lang M (2017). \dQuote{A Multicriteria Approach to Find Predictive and Sparse Models with Stable Feature Selection for High-Dimensional Data.} \emph{Computational and Mathematical Methods in Medicine}, \bold{2017}, 1--18. \doi{10.1155/2017/7907163}. Bommert A, Lang M (2021). \dQuote{stabm: Stability Measures for Feature Selection.} \emph{Journal of Open Source Software}, \bold{6}(59), 3010. \doi{10.21105/joss.03010}. } \seealso{ Package \CRANpkg{stabm} which implements many more stability measures with included correction for chance. Other Similarity Measures: \code{\link{jaccard}()} } \concept{Similarity Measures} \concept{similarity_measure} mlr3measures/man/confusion_matrix.Rd0000644000176200001440000000344614661131133017333 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/confusion_matrix.R \name{confusion_matrix} \alias{confusion_matrix} \title{Calculate Binary Confusion Matrix} \usage{ confusion_matrix(truth, response, positive, na_value = NaN, relative = FALSE) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{relative}{(\code{logical(1)})\cr If \code{TRUE}, the returned confusion matrix contains relative frequencies instead of absolute frequencies.} } \value{ List with two elements: \itemize{ \item \code{matrix} stores the calculated confusion matrix. \item \code{measures} stores the metrics as named numeric vector. } } \description{ Calculates the confusion matrix for a binary classification problem once and then calculates all binary confusion measures of this package. } \details{ The binary confusion matrix is defined as \deqn{ \begin{pmatrix} TP & FP \\ FN & TN \end{pmatrix}. }{ matrix(TP, FP, FN, TN). } If \code{relative = TRUE}, all values are divided by \eqn{n}. } \examples{ set.seed(123) lvls = c("a", "b") truth = factor(sample(lvls, 20, replace = TRUE), levels = lvls) response = factor(sample(lvls, 20, replace = TRUE), levels = lvls) confusion_matrix(truth, response, positive = "a") confusion_matrix(truth, response, positive = "a", relative = TRUE) confusion_matrix(truth, response, positive = "b") } mlr3measures/man/rrse.Rd0000644000176200001440000000421115056260067014716 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_rrse.R \name{rrse} \alias{rrse} \title{Root Relative Squared Error} \usage{ rrse(truth, response, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Root Relative Squared Error is defined as \deqn{ \sqrt{\frac{\sum_{i=1}^n \left( t_i - r_i \right)^2}{\sum_{i=1}^n \left( t_i - \bar{t} \right)^2}}, }{ sqrt(sum((t - r)^2) / sum((t - mean(t))^2)), } where \eqn{\bar{t} = \sum_{i=1}^n t_i}. Can be interpreted as root of the squared error of the predictions relative to a naive model predicting the mean. This measure is undefined for constant \eqn{t}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) rrse(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/mbrier.Rd0000644000176200001440000000446015170123771015226 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_mbrier.R \name{mbrier} \alias{mbrier} \title{Multiclass Brier Score} \usage{ mbrier(truth, prob, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{prob}{(\code{matrix()})\cr Matrix of predicted probabilities, each column is a vector of probabilities for a specific class label. Columns must be named with levels of \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted probabilities in multiclass classification tasks. } \details{ Brier score for multi-class classification problems with \eqn{k} labels defined as \deqn{ \frac{1}{n} \sum_{i=1}^n \sum_{j=1}^k (I_{ij} - p_{ij})^2. }{ } \eqn{I_{ij}}{I_ij} is 1 if observation \eqn{x_i} has true label \eqn{j}, and 0 otherwise. \eqn{p_{ij}}{p_ij} is the probability that observation \eqn{x_i} belongs to class \eqn{j}. Note that there also is the more common definition of the Brier score for binary classification problems in \code{\link[=bbrier]{bbrier()}}. } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range: \eqn{[0, 2]}{[0, 2]} \item Minimize: \code{TRUE} \item Required prediction: \code{prob} } } \examples{ set.seed(1) lvls = c("a", "b", "c") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) prob = matrix(runif(3 * 10), ncol = 3) colnames(prob) = levels(truth) mbrier(truth, prob) } \references{ Brier GW (1950). \dQuote{Verification of forecasts expressed in terms of probability.} \emph{Monthly Weather Review}, \bold{78}(1), 1--3. \doi{10.1175/1520-0493(1950)078<0001:vofeit>2.0.co;2}. } \seealso{ Other Classification Measures: \code{\link{acc}()}, \code{\link{bacc}()}, \code{\link{ce}()}, \code{\link{logloss}()}, \code{\link{mauc_aunu}()}, \code{\link{mcc}()}, \code{\link{zero_one}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/se.Rd0000644000176200001440000000330615056260067014356 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_se.R \name{se} \alias{se} \title{Squared Error (per observation)} \usage{ se(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(length(truth))}. } \description{ Measure to compare true observed response with predicted response in regression tasks. Note that this is an unaggregated measure, returning the losses per observation. } \details{ Calculates the per-observation squared error as \deqn{ \left( t_i - r_i \right)^2. }{ (t - r)^2. } } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range (per observation): \eqn{[0, \infty)}{[0, Inf)} \item Minimize (per observation): \code{TRUE} \item Required prediction: \code{response} } } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/zero_one.Rd0000644000176200001440000000303714661137565015577 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_zero_one.R \name{zero_one} \alias{zero_one} \alias{one_zero} \title{Zero-One Classification Loss (per observation)} \usage{ zero_one(truth, response, ...) one_zero(truth, response, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the same levels and length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(length(truth))}. } \description{ Calculates the per-observation 0/1 (zero-one) loss as \deqn{ \mathbf{1} (t_i \neq r_1). }{ 1(t != r). } The 1/0 (one-zero) loss is equal to 1 - zero-one and calculated as \deqn{ \mathbf{1} (t_i = r_i). }{ 1(t_i = r_i). } Measure to compare true observed labels with predicted labels in multiclass classification tasks. Note that this is an unaggregated measure, returning the losses per observation. } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range (per observation): \eqn{[0, 1]}{[0, 1]} \item Minimize (per observation): \code{TRUE} \item Required prediction: \code{response} } } \seealso{ Other Classification Measures: \code{\link{acc}()}, \code{\link{bacc}()}, \code{\link{ce}()}, \code{\link{logloss}()}, \code{\link{mauc_aunu}()}, \code{\link{mbrier}()}, \code{\link{mcc}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/maxae.Rd0000644000176200001440000000325215056260067015042 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_maxae.R \name{maxae} \alias{maxae} \title{Max Absolute Error} \usage{ maxae(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Max Absolute Error is defined as \deqn{ \max \left( \left| t_i - r_i \right| \right). }{ max(abs(t - r)). } } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) maxae(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/rmse.Rd0000644000176200001440000000377415056260067014726 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_rmse.R \name{rmse} \alias{rmse} \title{Root Mean Squared Error} \usage{ rmse(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Root Mean Squared Error is defined as \deqn{ \sqrt{\frac{1}{n} \sum_{i=1}^n w_i \left( t_i - r_i \right)^2}, }{ sqrt(weighted.mean((t - r)^2, w)), } where \eqn{w_i} are normalized sample weights. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) rmse(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/fpr.Rd0000644000176200001440000000460015170123773014533 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_fpr.R \name{fpr} \alias{fpr} \title{False Positive Rate} \usage{ fpr(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The False Positive Rate is defined as \deqn{ \frac{\mathrm{FP}}{\mathrm{FP} + \mathrm{TN}}. }{ FP / (FP + TN). } Also know as fall out or probability of false alarm. This measure is undefined if FP + TN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) fpr(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/rae.Rd0000644000176200001440000000414015056260067014513 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_rae.R \name{rae} \alias{rae} \title{Relative Absolute Error} \usage{ rae(truth, response, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Relative Absolute Error is defined as \deqn{ \frac{\sum_{i=1}^n \left| t_i - r_i \right|}{\sum_{i=1}^n \left| t_i - \bar{t} \right|}, }{ sum((t - r)^2) / sum((t - mean(t))^2), } where \eqn{\bar{t} = \sum_{i=1}^n t_i}. This measure is undefined for constant \eqn{t}. Can be interpreted as absolute error of the predictions relative to a naive model predicting the mean. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) rae(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/fdr.Rd0000644000176200001440000000451515170123773014524 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_fdr.R \name{fdr} \alias{fdr} \title{False Discovery Rate} \usage{ fdr(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The False Discovery Rate is defined as \deqn{ \frac{\mathrm{FP}}{\mathrm{TP} + \mathrm{FP}}. }{ FP / (TP + FP). } This measure is undefined if TP + FP = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) fdr(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/binary_params.Rd0000644000176200001440000000225114472156310016570 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/roxygen.R \name{binary_params} \alias{binary_params} \title{Binary Classification Parameters} \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{prob}{(\code{numeric()})\cr Predicted probability for positive class. Must have exactly same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \description{ Binary Classification Parameters } \keyword{internal} mlr3measures/man/rmsle.Rd0000644000176200001440000000450715056260067015075 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_rmsle.R \name{rmsle} \alias{rmsle} \title{Root Mean Squared Log Error} \usage{ rmsle(truth, response, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Root Mean Squared Log Error is defined as \deqn{ \sqrt{\frac{1}{n} \sum_{i=1}^n w_i \left( \ln (1 + t_i) - \ln (1 + r_i) \right)^2}, }{ sqrt(weighted.mean((log(1 + t) - log(1 + r))^2, w)), } where \eqn{w_i} are normalized sample weights. This measure is undefined if any element of \eqn{t} or \eqn{r} is less than or equal to \eqn{-1}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) rmsle(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/fp.Rd0000644000176200001440000000427015170123773014354 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_fp.R \name{fp} \alias{fp} \title{False Positives} \usage{ fp(truth, response, positive, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ This measure counts the false positives (type 1 error), i.e. the number of predictions indicating a positive class label while in fact it is negative. This is sometimes also called a "false alarm". } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) fp(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/mcc.Rd0000644000176200001440000000704715170376754014527 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_mcc.R \name{mcc} \alias{mcc} \title{Matthews Correlation Coefficient} \usage{ mcc(truth, response, positive = NULL, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the same levels and length as \code{truth}.} \item{positive}{(\code{character(1)}) Name of the positive class in case of binary classification.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in multiclass classification tasks. } \details{ In the binary case, the Matthews Correlation Coefficient is defined as \deqn{ \frac{\mathrm{TP} \cdot \mathrm{TN} - \mathrm{FP} \cdot \mathrm{FN}}{\sqrt{(\mathrm{TP} + \mathrm{FP}) (\mathrm{TP} + \mathrm{FN}) (\mathrm{TN} + \mathrm{FP}) (\mathrm{TN} + \mathrm{FN})}}, }{ (TP * TN - FP * FN) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)), } where \eqn{TP}, \eqn{FP}, \eqn{TN}, \eqn{TP} are the number of true positives, false positives, true negatives, and false negatives respectively. In the multi-class case, the Matthews Correlation Coefficient is defined for a multi-class confusion matrix \eqn{C} with \eqn{K} classes: \deqn{ \frac{c \cdot s - \sum_k^K p_k \cdot t_k}{\sqrt{(s^2 - \sum_k^K p_k^2) \cdot (s^2 - \sum_k^K t_k^2)}}, }{ (c * s - sum(pk * tk)) / sqrt((s^2 - sum(pk^2)) * (s^2 - sum(tk^2))), } where \itemize{ \item \eqn{s = \sum_i^K \sum_j^K C_{ij}}: total number of samples \item \eqn{c = \sum_k^K C_{kk}}: total number of correctly predicted samples \item \eqn{t_k = \sum_i^K C_{ik}}: number of predictions for each class \eqn{k} \item \eqn{p_k = \sum_j^K C_{kj}}: number of true occurrences for each class \eqn{k}. } The above formula is undefined if any of the four sums in the denominator is 0 in the binary case, and more generally if either \eqn{s^2 - \sum_k^K p_k^2} or \eqn{s^2 - \sum_k^K t_k^2)} is equal to 0. The denominator is then set to 1. When there are more than two classes, the MCC will no longer range between -1 and +1. Instead, the minimum value will be between -1 and 0 depending on the true distribution. The maximum value is always +1. } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range: \eqn{[-1, 1]}{[-1, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b", "c") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) mcc(truth, response) } \references{ \url{https://en.wikipedia.org/wiki/Phi_coefficient} Matthews BW (1975). \dQuote{Comparison of the predicted and observed secondary structure of T4 phage lysozyme.} \emph{Biochimica et Biophysica Acta (BBA) - Protein Structure}, \bold{405}(2), 442--451. \doi{10.1016/0005-2795(75)90109-9}. } \seealso{ Other Classification Measures: \code{\link{acc}()}, \code{\link{bacc}()}, \code{\link{ce}()}, \code{\link{logloss}()}, \code{\link{mauc_aunu}()}, \code{\link{mbrier}()}, \code{\link{zero_one}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/auc.Rd0000644000176200001440000000702015170370742014513 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_auc.R \name{auc} \alias{auc} \title{Area Under the ROC Curve} \usage{ auc(truth, prob, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{prob}{(\code{numeric()})\cr Predicted probability for positive class. Must have exactly same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted probabilities in binary classification tasks. } \details{ Computes the area under the Receiver Operator Characteristic (ROC) curve. The AUC can be interpreted as the probability that a randomly chosen positive observation has a higher predicted probability than a randomly chosen negative observation. For \eqn{n^+} positive and \eqn{n^-} negative observations with \eqn{R_i^+} the rank of the \eqn{i}-th positive observation's predicted probability (average ranks for ties), the AUC is estimated as \deqn{ \widehat{\operatorname{AUC}} = \frac{1}{n^+ n^-} \left( \sum_{i=1}^{n^+} R_i^+ \; - \; \frac{n^+(n^+ + 1)}{2} \right). }{ AUC = (sum(R_pos) - n_pos * (n_pos + 1) / 2) / (n_pos * n_neg). } If \code{sample_weights} are provided, let \eqn{w_i^+} be the weight of the \eqn{i}-th positive observation with predicted probability \eqn{p_i^+}, \eqn{W^+ = \sum_i w_i^+}, and \eqn{W^-} the total weight of the negative observations. Define the weighted Mann-Whitney contribution of positive observation \eqn{i} as \eqn{U_i^w = W_{< p_i^+}^- + \tfrac{1}{2} W_{= p_i^+}^-}, i.e. the total weight of negative observations with a smaller predicted probability plus half the weight of negatives tied with \eqn{p_i^+}. The weighted AUC is then \deqn{ \widehat{\operatorname{AUC}}_w = \frac{1}{W^+ W^-} \sum_{i=1}^{n^+} w_i^+ U_i^w. }{ AUC_w = sum(w_pos * U_pos) / (W_pos * W_neg). } This measure is undefined if the true values are either all positive or all negative. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{prob} } } \examples{ truth = factor(c("a", "a", "a", "b")) prob = c(.6, .7, .1, .4) auc(truth, prob, "a") } \references{ Youden WJ (1950). \dQuote{Index for rating diagnostic tests.} \emph{Cancer}, \bold{3}(1), 32--35. \doi{10.1002/1097-0142(1950)3:1<32::aid-cncr2820030106>3.0.co;2-3}. } \seealso{ Other Binary Classification Measures: \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/ae.Rd0000644000176200001440000000330715056260067014335 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_ae.R \name{ae} \alias{ae} \title{Absolute Error (per observation)} \usage{ ae(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(length(truth))}. } \description{ Measure to compare true observed response with predicted response in regression tasks. Note that this is an unaggregated measure, returning the losses per observation. } \details{ Calculates the per-observation absolute error as \deqn{ \left| t_i - r_i \right|. }{ abs(t - r). } } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range (per observation): \eqn{[0, \infty)}{[0, Inf)} \item Minimize (per observation): \code{TRUE} \item Required prediction: \code{response} } } \seealso{ Other Regression Measures: \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/similarity_params.Rd0000644000176200001440000000122214472156310017467 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/roxygen.R \name{similarity_params} \alias{similarity_params} \title{Similarity Parameters} \arguments{ \item{sets}{(\code{list()})\cr List of character or integer vectors. \code{sets} must have at least 2 elements.} \item{p}{(\code{integer(1)})\cr Total number of possible elements.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \description{ Similarity Parameters } \keyword{internal} mlr3measures/man/jaccard.Rd0000644000176200001440000000423014472156310015327 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/similarity_jaccard.R \name{jaccard} \alias{jaccard} \title{Jaccard Similarity Index} \usage{ jaccard(sets, na_value = NaN, ...) } \arguments{ \item{sets}{(\code{list()})\cr List of character or integer vectors. \code{sets} must have at least 2 elements.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare two or more sets w.r.t. their similarity. } \details{ For two sets \eqn{A} and \eqn{B}, the Jaccard Index is defined as \deqn{ J(A, B) = \frac{|A \cap B|}{|A \cup B|}. }{ J(A, B) = length(intersect(A, B)) / length(union(A, B)). } If more than two sets are provided, the mean of all pairwise scores is calculated. This measure is undefined if two or more sets are empty. } \section{Meta Information}{ \itemize{ \item Type: \code{"similarity"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} } } \examples{ set.seed(1) sets = list( sample(letters[1:3], 1), sample(letters[1:3], 2) ) jaccard(sets) } \references{ Jaccard, Paul (1901). \dQuote{Étude comparative de la distribution florale dans une portion des Alpes et du Jura.} \emph{Bulletin de la Société Vaudoise des Sciences Naturelles}, \bold{37}, 547-579. \doi{10.5169/SEALS-266450}. Bommert A, Rahnenführer J, Lang M (2017). \dQuote{A Multicriteria Approach to Find Predictive and Sparse Models with Stable Feature Selection for High-Dimensional Data.} \emph{Computational and Mathematical Methods in Medicine}, \bold{2017}, 1--18. \doi{10.1155/2017/7907163}. Bommert A, Lang M (2021). \dQuote{stabm: Stability Measures for Feature Selection.} \emph{Journal of Open Source Software}, \bold{6}(59), 3010. \doi{10.21105/joss.03010}. } \seealso{ Package \CRANpkg{stabm} which implements many more stability measures with included correction for chance. Other Similarity Measures: \code{\link{phi}()} } \concept{Similarity Measures} \concept{similarity_measure} mlr3measures/man/mape.Rd0000644000176200001440000000500315056260067014665 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_mape.R \name{mape} \alias{mape} \title{Mean Absolute Percent Error} \usage{ mape(truth, response, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Mean Absolute Percent Error is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i \left| \frac{ t_i - r_i}{t_i} \right|, }{ weighted.mean(abs((t - r) / t), w), } where \eqn{w_i} are normalized sample weights. This measure is undefined if any element of \eqn{t} is \eqn{0}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) mape(truth, response) } \references{ de Myttenaere, Arnaud, Golden, Boris, Le Grand, Bénédicte, Rossi, Fabrice (2016). \dQuote{Mean Absolute Percentage Error for regression models.} \emph{Neurocomputing}, \bold{192}, 38-48. ISSN 0925-2312, \doi{10.1016/j.neucom.2015.12.114}. } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/sse.Rd0000644000176200001440000000375715056260067014553 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_sse.R \name{sse} \alias{sse} \title{Sum of Squared Errors} \usage{ sse(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. Weights for this function are not normalized. Defaults to sample weights 1.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Sum of Squared Errors is defined as \deqn{ \sum_{i=1}^n w_i \left( t_i - r_i \right)^2. }{ sum(w * (t - r)^2). } where \eqn{w_i} are unnormalized weights for each observation \eqn{x_i}, defaulting to 1. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) sse(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/acc.Rd0000644000176200001440000000340615056012454014471 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_acc.R \name{acc} \alias{acc} \title{Classification Accuracy} \usage{ acc(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the same levels and length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in multiclass classification tasks. } \details{ The Classification Accuracy is defined as \deqn{ \sum_{i=1}^n w_i \mathbf{1} \left( t_i = r_i \right), }{sum(wi * 1(ti = ri))} where \eqn{w_i} are weights normalized to sum to 1 for all observations \eqn{x_i}. } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b", "c") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) acc(truth, response) } \seealso{ Other Classification Measures: \code{\link{bacc}()}, \code{\link{ce}()}, \code{\link{logloss}()}, \code{\link{mauc_aunu}()}, \code{\link{mbrier}()}, \code{\link{mcc}()}, \code{\link{zero_one}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/ppv.Rd0000644000176200001440000000531215170123773014552 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_ppv.R \name{ppv} \alias{ppv} \alias{precision} \title{Positive Predictive Value} \usage{ ppv(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) precision( truth, response, positive, sample_weights = NULL, na_value = NaN, ... ) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The Positive Predictive Value is defined as \deqn{ \frac{\mathrm{TP}}{\mathrm{TP} + \mathrm{FP}}. }{ TP / (TP + FP). } Also know as "precision". This measure is undefined if TP + FP = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) ppv(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} Goutte C, Gaussier E (2005). \dQuote{A Probabilistic Interpretation of Precision, Recall and F-Score, with Implication for Evaluation.} In \emph{Lecture Notes in Computer Science}, 345--359. \doi{10.1007/978-3-540-31865-1_25}. } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/sle.Rd0000644000176200001440000000335015056260067014531 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_sle.R \name{sle} \alias{sle} \title{Squared Log Error (per observation)} \usage{ sle(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(length(truth))}. } \description{ Calculates the per-observation squared error as \deqn{ \left( \ln (1 + t_i) - \ln (1 + r_i) \right)^2. }{ (log(1 + t) - log(1 + r))^2. } Measure to compare true observed response with predicted response in regression tasks. Note that this is an unaggregated measure, returning the losses per observation. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range (per observation): \eqn{[0, \infty)}{[0, Inf)} \item Minimize (per observation): \code{TRUE} \item Required prediction: \code{response} } } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/macros/0000755000176200001440000000000014472156310014736 5ustar liggesusersmlr3measures/man/macros/cite.Rd0000644000176200001440000000012214472156310016144 0ustar liggesusers\newcommand{\cite}{\Sexpr[results=rd,stage=build]{mlr3measures:::cite_bib("#1")}} mlr3measures/man/regr_params.Rd0000644000176200001440000000161514472156310016246 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/roxygen.R \name{regr_params} \alias{regr_params} \title{Regression Parameters} \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \description{ Regression Parameters } \keyword{internal} mlr3measures/man/linex.Rd0000644000176200001440000000473015056260067015070 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_linex.R \name{linex} \alias{linex} \title{Linear-Exponential Loss (per observation)} \usage{ linex(truth, response, a = -1, b = 1, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{a}{(\code{numeric(1)})\cr Shape parameter controlling asymmetry. Negative values penalize overestimation more, positive values penalize underestimation more. As \code{a} approaches 0, the loss resembles squared error loss. Default is \code{-1}.} \item{b}{(\code{numeric(1)})\cr Positive scaling factor for the loss. Larger values increase the loss magnitude. Default is \code{1}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(length(truth))}. } \description{ Measure to compare true observed response with predicted response in regression tasks. Note that this is an unaggregated measure, returning the losses per observation. } \details{ The Linear-Exponential Loss is defined as \deqn{ b (\exp (t_i - r_i) - a (t_i - r_i) - 1), }{ } where \eqn{a \neq 0, b > 0}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range (per observation): \eqn{[0, \infty)}{[0, Inf)} \item Minimize (per observation): \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) linex(truth, response) } \references{ Varian, R. H (1975). \dQuote{A Bayesian Approach to Real Estate Assessment.} In Fienberg SE, Zellner A (eds.), \emph{Studies in Bayesian Econometrics and Statistics: In Honor of Leonard J. Savage}, 195--208. North-Holland, Amsterdam. } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/fnr.Rd0000644000176200001440000000454515170123773014541 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_fnr.R \name{fnr} \alias{fnr} \title{False Negative Rate} \usage{ fnr(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The False Negative Rate is defined as \deqn{ \frac{\mathrm{FN}}{\mathrm{TP} + \mathrm{FN}}. }{ FN / (TP + FN). } Also know as "miss rate". This measure is undefined if TP + FN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) fnr(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/fomr.Rd0000644000176200001440000000451715170123773014716 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_fomr.R \name{fomr} \alias{fomr} \title{False Omission Rate} \usage{ fomr(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The False Omission Rate is defined as \deqn{ \frac{\mathrm{FN}}{\mathrm{FN} + \mathrm{TN}}. }{ FN / (FN + TN). } This measure is undefined if FN + TN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) fomr(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/mauc_aunu.Rd0000644000176200001440000001023214664404121015713 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_auc.R \name{mauc_aunu} \alias{mauc_aunu} \alias{mauc_aunp} \alias{mauc_au1u} \alias{mauc_au1p} \alias{mauc_mu} \title{Multiclass AUC Scores} \usage{ mauc_aunu(truth, prob, na_value = NaN, ...) mauc_aunp(truth, prob, na_value = NaN, ...) mauc_au1u(truth, prob, na_value = NaN, ...) mauc_au1p(truth, prob, na_value = NaN, ...) mauc_mu(truth, prob, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{prob}{(\code{matrix()})\cr Matrix of predicted probabilities, each column is a vector of probabilities for a specific class label. Columns must be named with levels of \code{truth}.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted probabilities in multiclass classification tasks. } \details{ Multiclass AUC measures. \itemize{ \item \emph{AUNU}: AUC of each class against the rest, using the uniform class distribution. Computes the AUC treating a \code{c}-dimensional classifier as \code{c} two-dimensional 1-vs-rest classifiers, where classes are assumed to have uniform distribution, in order to have a measure which is independent of class distribution change (Fawcett 2001). \item \emph{AUNP}: AUC of each class against the rest, using the a-priori class distribution. Computes the AUC treating a \code{c}-dimensional classifier as \code{c} two-dimensional 1-vs-rest classifiers, taking into account the prior probability of each class (Fawcett 2001). \item \emph{AU1U}: AUC of each class against each other, using the uniform class distribution. Computes something like the AUC of \code{c(c - 1)} binary classifiers (all possible pairwise combinations). See Hand (2001) for details. \item \emph{AU1P}: AUC of each class against each other, using the a-priori class distribution. Computes something like AUC of \code{c(c - 1)} binary classifiers while considering the a-priori distribution of the classes as suggested in Ferri (2009). Note we deviate from the definition in Ferri (2009) by a factor of \code{c}. \item \emph{MU}: Multiclass AUC as defined in Kleinman and Page (2019). This measure is an average of the pairwise AUCs between all classes. The measure was tested against the Python implementation by \href{https://github.com/kleimanr/auc_mu}{Ross Kleinman}. } } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{prob} } } \examples{ set.seed(1) lvls = c("a", "b", "c") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) prob = matrix(runif(3 * 10), ncol = 3) colnames(prob) = levels(truth) mauc_aunu(truth, prob) } \references{ Fawcett, Tom (2001). \dQuote{Using rule sets to maximize ROC performance.} In \emph{Proceedings 2001 IEEE international conference on data mining}, 131--138. IEEE. Ferri, César, Hernández-Orallo, José, Modroiu, R (2009). \dQuote{An experimental comparison of performance measures for classification.} \emph{Pattern Recognition Letters}, \bold{30}(1), 27--38. \doi{10.1016/j.patrec.2008.08.010}. Hand, J D, Till, J R (2001). \dQuote{A simple generalisation of the area under the ROC curve for multiple class classification problems.} \emph{Machine learning}, \bold{45}(2), 171--186. Kleiman R, Page D (2019). \dQuote{AUC mu: A Performance Metric for Multi-Class Machine Learning Models.} In Chaudhuri, Kamalika, Salakhutdinov, Ruslan (eds.), \emph{Proceedings of the 36th International Conference on Machine Learning}, volume 97 series Proceedings of Machine Learning Research, 3439--3447. PMLR. } \seealso{ Other Classification Measures: \code{\link{acc}()}, \code{\link{bacc}()}, \code{\link{ce}()}, \code{\link{logloss}()}, \code{\link{mbrier}()}, \code{\link{mcc}()}, \code{\link{zero_one}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/medse.Rd0000644000176200001440000000330315056260067015041 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_medse.R \name{medse} \alias{medse} \title{Median Squared Error} \usage{ medse(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Median Squared Error is defined as \deqn{ \mathop{\mathrm{median}} \left[ \left( t_i - r_i \right)^2 \right]. }{ median((t - r)^2). } } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) medse(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/rse.Rd0000644000176200001440000000414215056260067014537 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_rse.R \name{rse} \alias{rse} \title{Relative Squared Error} \usage{ rse(truth, response, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Relative Squared Error is defined as \deqn{ \frac{\sum_{i=1}^n \left( t_i - r_i \right)^2}{\sum_{i=1}^n \left( t_i - \bar{t} \right)^2}, }{ sum((t - r)^2) / sum((t - mean(t))^2), } where \eqn{\bar{t} = \sum_{i=1}^n t_i}. Can be interpreted as squared error of the predictions relative to a naive model predicting the mean. This measure is undefined for constant \eqn{t}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) rse(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/ktau.Rd0000644000176200001440000000424115170376754014722 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_ktau.R \name{ktau} \alias{ktau} \title{Kendall's tau} \usage{ ktau(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ Kendall's tau is defined as Kendall's rank correlation coefficient between truth and response. It is defined as \deqn{ \tau = \frac{(\mathrm{number of concordant pairs)} - (\mathrm{number of discordant pairs)}}{\mathrm{(number of pairs)}} }{ t = (number of concordant pairs) - (number of discordant pairs) / (number of pairs)} Calls \code{\link[stats:cor]{stats::cor()}} with \code{method} set to \code{"kendall"}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[-1, 1]}{[-1, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) ktau(truth, response) } \references{ Rosset S, Perlich C, Zadrozny B (2006). \dQuote{Ranking-based evaluation of regression models.} \emph{Knowledge and Information Systems}, \bold{12}(3), 331--353. \doi{10.1007/s10115-006-0037-3}. } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/ape.Rd0000644000176200001440000000336315056260067014517 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_ape.R \name{ape} \alias{ape} \title{Absolute Percentage Error (per observation)} \usage{ ape(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(length(truth))}. } \description{ Measure to compare true observed response with predicted response in regression tasks. Note that this is an unaggregated measure, returning the losses per observation. } \details{ Calculates the per-observation absolute percentage error as \deqn{ \left| \frac{ t_i - r_i}{t_i} \right|. }{ abs((t - r) / t). } } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range (per observation): \eqn{[0, \infty)}{[0, Inf)} \item Minimize (per observation): \code{TRUE} \item Required prediction: \code{response} } } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/npv.Rd0000644000176200001440000000453015170123773014551 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_npv.R \name{npv} \alias{npv} \title{Negative Predictive Value} \usage{ npv(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The Negative Predictive Value is defined as \deqn{ \frac{\mathrm{TN}}{\mathrm{FN} + \mathrm{TN}}. }{ TN / (FN + TN). } This measure is undefined if FN + TN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) npv(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/gmean.Rd0000644000176200001440000000512415170123773015035 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_gmean.R \name{gmean} \alias{gmean} \title{Geometric Mean of Recall and Specificity} \usage{ gmean(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ Calculates the geometric mean of \code{\link[=recall]{recall()}} R and \code{\link[=specificity]{specificity()}} S as \deqn{ \sqrt{\mathrm{R} \cdot \mathrm{S}}. }{ sqrt(R * S) } This measure is undefined if recall or specificity is undefined, i.e. if TP + FN = 0 or if FP + TN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) gmean(truth, response, positive = "a") } \references{ He H, Garcia EA (2009). \dQuote{Learning from Imbalanced Data.} \emph{IEEE Transactions on knowledge and data engineering}, \bold{21}(9), 1263--1284. \doi{10.1109/TKDE.2008.239}. } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/bacc.Rd0000644000176200001440000000535114661137565014650 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_bacc.R \name{bacc} \alias{bacc} \title{Balanced Accuracy} \usage{ bacc(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the same levels and length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in multiclass classification tasks. } \details{ The Balanced Accuracy computes the weighted balanced accuracy, suitable for imbalanced data sets. It is defined analogously to the definition in \href{https://scikit-learn.org/}{sklearn}. First, all sample weights \eqn{w_i} are normalized per class so that each class has the same influence: \deqn{ \hat{w}_i = \frac{w_i}{\sum_{j=1}^n w_j \cdot \mathbf{1}(t_j = t_i)}. }{ w_hat[i] = w[i] / sum((t == t[i]) * w[i]). } The Balanced Accuracy is then calculated as \deqn{ \frac{1}{\sum_{i=1}^n \hat{w}_i} \sum_{i=1}^n \hat{w}_i \cdot \mathbf{1}(r_i = t_i). }{ 1 / sum(w_hat) * sum((r == t) * w_hat). } This definition is equivalent to \code{\link[=acc]{acc()}} with class-balanced sample weights. } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b", "c") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) bacc(truth, response) } \references{ Brodersen KH, Ong CS, Stephan KE, Buhmann JM (2010). \dQuote{The Balanced Accuracy and Its Posterior Distribution.} In \emph{2010 20th International Conference on Pattern Recognition}. \doi{10.1109/icpr.2010.764}. Guyon I, Bennett K, Cawley G, Escalante HJ, Escalera S, Ho TK, Macia N, Ray B, Saeed M, Statnikov A, Viegas E (2015). \dQuote{Design of the 2015 ChaLearn AutoML challenge.} In \emph{2015 International Joint Conference on Neural Networks (IJCNN)}. \doi{10.1109/ijcnn.2015.7280767}. } \seealso{ Other Classification Measures: \code{\link{acc}()}, \code{\link{ce}()}, \code{\link{logloss}()}, \code{\link{mauc_aunu}()}, \code{\link{mbrier}()}, \code{\link{mcc}()}, \code{\link{zero_one}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/obs_logloss.Rd0000644000176200001440000000050715111057476016274 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_logloss.R \name{obs_logloss} \alias{obs_logloss} \title{Observation-wise Log Loss} \usage{ obs_logloss(truth, prob, eps = 1e-15, ...) } \description{ Observation-wise loss function for \code{\link[=logloss]{logloss()}}. } \keyword{internal} mlr3measures/man/mae.Rd0000644000176200001440000000374215056260067014515 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_mae.R \name{mae} \alias{mae} \title{Mean Absolute Error} \usage{ mae(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Mean Absolute Error is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i \left| t_i - r_i \right|, }{ weighted.mean(abs(t - r), w), } where \eqn{w_i} are normalized sample weights. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) mae(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/smape.Rd0000644000176200001440000000441515170376754015066 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_smape.R \name{smape} \alias{smape} \title{Symmetric Mean Absolute Percent Error} \usage{ smape(truth, response, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Symmetric Mean Absolute Percent Error is defined as \deqn{ \frac{2}{n} \sum_{i=1}^n \frac{\left| t_i - r_i \right|}{\left| t_i \right| + \left| r_i \right|}. }{ 2 * mean(abs(t - r) / (abs(t) + abs(r))). } This measure is undefined if any \eqn{|t| + |r|} is equal to \eqn{0}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, 2]}{[0, 2]} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) smape(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/dor.Rd0000644000176200001440000000455115170123773014535 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_dor.R \name{dor} \alias{dor} \title{Diagnostic Odds Ratio} \usage{ dor(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The Diagnostic Odds Ratio is defined as \deqn{ \frac{\mathrm{TP}/\mathrm{FP}}{\mathrm{FN}/\mathrm{TN}}. }{ (TP/FP) / (FN/TN). } This measure is undefined if FP = 0 or FN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) dor(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/logloss.Rd0000644000176200001440000000430315170376754015437 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_logloss.R \name{logloss} \alias{logloss} \title{Log Loss} \usage{ logloss(truth, prob, sample_weights = NULL, eps = 1e-15, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{prob}{(\code{matrix()})\cr Matrix of predicted probabilities, each column is a vector of probabilities for a specific class label. Columns must be named with levels of \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{eps}{(\code{numeric(1)})\cr Probabilities are clipped to \code{max(eps, min(1 - eps, p))}. Otherwise the measure would be undefined for probabilities \code{p = 0} and \code{p = 1}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted probabilities in multiclass classification tasks. } \details{ The Log Loss (a.k.a Bernoulli Loss, Logistic Loss, Cross-Entropy Loss) is defined as \deqn{ -\frac{1}{n} \sum_{i=1}^n w_i \log \left( p_i \right ) }{ -weighted.mean(log(p), w) } where \eqn{p_i}{p} is the probability for the true class of observation \eqn{i}, and \eqn{w_i} are normalized weights for each observation \eqn{x_i}. } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{prob} } } \examples{ set.seed(1) lvls = c("a", "b", "c") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) prob = matrix(runif(3 * 10), ncol = 3, dimnames = list(NULL, lvls)) prob = t(apply(prob, 1, function(x) x / sum(x))) logloss(truth, prob) } \seealso{ Other Classification Measures: \code{\link{acc}()}, \code{\link{bacc}()}, \code{\link{ce}()}, \code{\link{mauc_aunu}()}, \code{\link{mbrier}()}, \code{\link{mcc}()}, \code{\link{zero_one}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/prauc.Rd0000644000176200001440000000451615073220360015054 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_prauc.R \name{prauc} \alias{prauc} \title{Area Under the Precision-Recall Curve} \usage{ prauc(truth, prob, positive, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{prob}{(\code{numeric()})\cr Predicted probability for positive class. Must have exactly same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted probabilities in binary classification tasks. } \details{ Computes the area under the Precision-Recall curve (PRC). The PRC can be interpreted as the relationship between precision and recall (sensitivity), and is considered to be a more appropriate measure for unbalanced datasets than the ROC curve. The AUC-PRC is computed by integration of the piecewise function. This measure is undefined if the true values are either all positive or all negative. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{prob} } } \examples{ truth = factor(c("a", "a", "a", "b")) prob = c(.6, .7, .1, .4) prauc(truth, prob, "a") } \references{ Davis J, Goadrich M (2006). \dQuote{The relationship between precision-recall and ROC curves.} In \emph{Proceedings of the 23rd International Conference on Machine Learning}. ISBN 9781595933836. } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/tn.Rd0000644000176200001440000000423315170123773014367 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_tn.R \name{tn} \alias{tn} \title{True Negatives} \usage{ tn(truth, response, positive, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ This measure counts the true negatives, i.e. the number of predictions correctly indicating a negative class label. This is sometimes also called a "correct rejection". } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) tn(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/ce.Rd0000644000176200001440000000341414661137565014345 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/classif_ce.R \name{ce} \alias{ce} \title{Classification Error} \usage{ ce(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the same levels and length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the same levels and length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in multiclass classification tasks. } \details{ The Classification Error is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i \mathbf{1} \left( t_i \neq r_i \right), }{ 1 / n * sum(wi * 1(ti != ri)), } where \eqn{w_i} are normalized weights for each observation \eqn{x_i}. } \section{Meta Information}{ \itemize{ \item Type: \code{"classif"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b", "c") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) ce(truth, response) } \seealso{ Other Classification Measures: \code{\link{acc}()}, \code{\link{bacc}()}, \code{\link{logloss}()}, \code{\link{mauc_aunu}()}, \code{\link{mbrier}()}, \code{\link{mcc}()}, \code{\link{zero_one}()} } \concept{Classification Measures} \concept{classification_measure} mlr3measures/man/measures.Rd0000644000176200001440000000371415170376754015606 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/measures.R \docType{data} \name{measures} \alias{measures} \title{Measure Registry} \format{ An object of class \code{MeasureEnv} (inherits from \code{environment}) of length 65. } \usage{ measures } \description{ The \code{\link[=environment]{environment()}} \code{measures} keeps track of all measures in this package. It stores meta information such as minimum, maximum or if the measure must be minimized or maximized. The following information is available for each measure: \itemize{ \item \code{id}: Name of the measure. \item \code{title}: Short descriptive title. \item \code{type}: \code{"binary"} for binary classification, \code{"classif"} for binary or multi-class classification, \code{"regr"} for regression and \code{"similarity"} for similarity measures. \item \code{lower}: lower bound. \item \code{upper}: upper bound. \item \code{predict_type}: prediction type the measure operates on. \code{"response"} corresponds to class labels for classification and the numeric response for regression. \code{"prob"} corresponds to class probabilities, provided as a matrix with class labels as column names. \code{"se"} corresponds to the vector of predicted standard errors for regression. \item \code{minimize}: If \code{TRUE} or \code{FALSE}, the objective is to minimize or maximize the measure, respectively. Can also be \code{NA}. \item \code{obs_loss}: Name of the function which is called to calculate the (unaggregated) loss per observation. \item \code{trafo}: Optional \code{list()} of length 2, containing a transformation \code{"fn"} and its derivative \code{"deriv"}. \item \code{aggregated}: If \code{TRUE}, this function aggregates the losses to a single numeric value. Otherwise, a vector of losses is returned. \item \code{sample_weights}: If \code{TRUE}, it is possible calculate a weighted measure. } } \examples{ names(measures) measures$tpr as.data.frame(measures) } \keyword{datasets} mlr3measures/man/rsq.Rd0000644000176200001440000000430215056260067014551 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_rsq.R \name{rsq} \alias{rsq} \title{R Squared} \usage{ rsq(truth, response, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ R Squared is defined as \deqn{ 1 - \frac{\sum_{i=1}^n \left( t_i - r_i \right)^2}{\sum_{i=1}^n \left( t_i - \bar{t} \right)^2}, }{ 1 - sum((t - r)^2) / sum((t - mean(t))^2), } where \eqn{\bar{t} = \sum_{i=1}^n t_i}. Also known as coefficient of determination or explained variation. Subtracts the \code{\link[=rse]{rse()}} from 1, hence it compares the squared error of the predictions relative to a naive model predicting the mean. This measure is undefined for constant \eqn{t}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{(-\infty, 1]}{(-Inf, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) rsq(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/pinball.Rd0000644000176200001440000000434315056260067015372 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_pinball.R \name{pinball} \alias{pinball} \title{Average Pinball Loss} \usage{ pinball(truth, response, sample_weights = NULL, alpha = 0.5, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{alpha}{\code{numeric(1)}\cr The quantile to compute the pinball loss.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The pinball loss for quantile regression is defined as \deqn{ \text{Average Pinball Loss} = \frac{1}{n} \sum_{i=1}^{n} w_{i} \begin{cases} q \cdot (t_i - r_i) & \text{if } t_i \geq r_i \\ (1 - q) \cdot (r_i - t_i) & \text{if } t_i < r_i \end{cases} } where \eqn{q} is the quantile and \eqn{w_i} are normalized sample weights. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{(-\infty, \infty)}{(-Inf, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) pinball(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/tp.Rd0000644000176200001440000000421515170123773014371 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_tp.R \name{tp} \alias{tp} \title{True Positives} \usage{ tp(truth, response, positive, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ This measure counts the true positives, i.e. the number of predictions correctly indicating a positive class label. This is sometimes also called a "hit". } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) tp(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/msle.Rd0000644000176200001440000000446115056260067014712 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_msle.R \name{msle} \alias{msle} \title{Mean Squared Log Error} \usage{ msle(truth, response, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Mean Squared Log Error is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i \left( \ln (1 + t_i) - \ln (1 + r_i) \right)^2, }{ weighted.mean((log(1 + t) - log(1 + r))^2, weights), } where \eqn{w_i} are normalized sample weights. This measure is undefined if any element of \eqn{t} or \eqn{r} is less than or equal to \eqn{-1}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) msle(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/pbias.Rd0000644000176200001440000000433515056260067015050 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_pbias.R \name{pbias} \alias{pbias} \title{Percent Bias} \usage{ pbias(truth, response, sample_weights = NULL, na_value = NaN, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Percent Bias is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i \frac{\left( r_i - t_i \right)}{\left| t_i \right|}, }{ weighted.mean((r - t) / abs(t), w), } where \eqn{w_i} are normalized sample weights. Good predictions score close to 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{(-\infty, \infty)}{(-Inf, Inf)} \item Minimize: \code{NA} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) pbias(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/fn.Rd0000644000176200001440000000431115170123773014346 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_fn.R \name{fn} \alias{fn} \title{False Negatives} \usage{ fn(truth, response, positive, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ This measure counts the false negatives (type 2 error), i.e. the number of predictions indicating a negative class label while in fact it is positive. This is sometimes also called a "miss" or an "underestimation". } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) fn(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/tpr.Rd0000644000176200001440000000551715170123773014561 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_tpr.R \name{tpr} \alias{tpr} \alias{recall} \alias{sensitivity} \title{True Positive Rate} \usage{ tpr(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) recall(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) sensitivity( truth, response, positive, sample_weights = NULL, na_value = NaN, ... ) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The True Positive Rate is defined as \deqn{ \frac{\mathrm{TP}}{\mathrm{TP} + \mathrm{FN}}. }{ TP / (TP + FN). } This is also know as "recall", "sensitivity", or "probability of detection". This measure is undefined if TP + FN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) tpr(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} Goutte C, Gaussier E (2005). \dQuote{A Probabilistic Interpretation of Precision, Recall and F-Score, with Implication for Evaluation.} In \emph{Lecture Notes in Computer Science}, 345--359. \doi{10.1007/978-3-540-31865-1_25}. } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/bbrier.Rd0000644000176200001440000000505615073220360015207 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_bbrier.R \name{bbrier} \alias{bbrier} \title{Binary Brier Score} \usage{ bbrier(truth, prob, positive, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{prob}{(\code{numeric()})\cr Predicted probability for positive class. Must have exactly same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted probabilities in binary classification tasks. } \details{ The Binary Brier Score is defined as \deqn{ \frac{1}{n} \sum_{i=1}^n w_i (I_i - p_i)^2, }{ weighted.mean(((t == positive) - p)^2, w), } where \eqn{w_i} are the sample weights, and \eqn{I_{i}} is 1 if observation \eqn{x_i} belongs to the positive class, and 0 otherwise. Note that this (more common) definition of the Brier score is equivalent to the original definition of the multi-class Brier score (see \code{\link[=mbrier]{mbrier()}}) divided by 2. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{TRUE} \item Required prediction: \code{prob} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) prob = runif(10) bbrier(truth, prob, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Brier_score} Brier GW (1950). \dQuote{Verification of forecasts expressed in terms of probability.} \emph{Monthly Weather Review}, \bold{78}(1), 1--3. \doi{10.1175/1520-0493(1950)078<0001:vofeit>2.0.co;2}. } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/srho.Rd0000644000176200001440000000366315056260067014730 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_srho.R \name{srho} \alias{srho} \title{Spearman's rho} \usage{ srho(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ Spearman's rho is defined as Spearman's rank correlation coefficient between truth and response. Calls \code{\link[stats:cor]{stats::cor()}} with \code{method} set to \code{"spearman"}. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[-1, 1]}{[-1, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) srho(truth, response) } \references{ Rosset S, Perlich C, Zadrozny B (2006). \dQuote{Ranking-based evaluation of regression models.} \emph{Knowledge and Information Systems}, \bold{12}(3), 331--353. \doi{10.1007/s10115-006-0037-3}. } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/sae.Rd0000644000176200001440000000376215056260067014525 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_sae.R \name{sae} \alias{sae} \title{Sum of Absolute Errors} \usage{ sae(truth, response, sample_weights = NULL, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. Weights for this function are not normalized. Defaults to sample weights 1.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Sum of Absolute Errors is defined as \deqn{ \sum_{i=1}^n w_i \left| t_i - r_i \right|. }{ sum(w * abs((t - r))). } where \eqn{w_i} are unnormalized weights for each observation \eqn{x_i}, defaulting to 1. } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) sae(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{maxse}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/tnr.Rd0000644000176200001440000000475515170123773014562 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_tnr.R \name{tnr} \alias{tnr} \alias{specificity} \title{True Negative Rate} \usage{ tnr(truth, response, positive, sample_weights = NULL, na_value = NaN, ...) specificity( truth, response, positive, sample_weights = NULL, na_value = NaN, ... ) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ The True Negative Rate is defined as \deqn{ \frac{\mathrm{TN}}{\mathrm{FP} + \mathrm{TN}}. }{ TN / (FP + TN). } Also know as "specificity" or "selectivity". This measure is undefined if FP + TN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) tnr(truth, response, positive = "a") } \references{ \url{https://en.wikipedia.org/wiki/Template:DiagnosticTesting_Diagram} } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fbeta}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/man/maxse.Rd0000644000176200001440000000323215056260067015062 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/regr_maxse.R \name{maxse} \alias{maxse} \title{Max Squared Error} \usage{ maxse(truth, response, ...) } \arguments{ \item{truth}{(\code{numeric()})\cr True (observed) values. Must have the same length as \code{response}.} \item{response}{(\code{numeric()})\cr Predicted response values. Must have the same length as \code{truth}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed response with predicted response in regression tasks. } \details{ The Max Squared Error is defined as \deqn{ \max \left( t_i - r_i \right)^2. }{ max((t - r)^2). } } \section{Meta Information}{ \itemize{ \item Type: \code{"regr"} \item Range: \eqn{[0, \infty)}{[0, Inf)} \item Minimize: \code{TRUE} \item Required prediction: \code{response} } } \examples{ set.seed(1) truth = 1:10 response = truth + rnorm(10) maxse(truth, response) } \seealso{ Other Regression Measures: \code{\link{ae}()}, \code{\link{ape}()}, \code{\link{bias}()}, \code{\link{ktau}()}, \code{\link{linex}()}, \code{\link{mae}()}, \code{\link{mape}()}, \code{\link{maxae}()}, \code{\link{medae}()}, \code{\link{medse}()}, \code{\link{mse}()}, \code{\link{msle}()}, \code{\link{pbias}()}, \code{\link{pinball}()}, \code{\link{rae}()}, \code{\link{rmse}()}, \code{\link{rmsle}()}, \code{\link{rrse}()}, \code{\link{rse}()}, \code{\link{rsq}()}, \code{\link{sae}()}, \code{\link{se}()}, \code{\link{sle}()}, \code{\link{smape}()}, \code{\link{srho}()}, \code{\link{sse}()} } \concept{Regression Measures} \concept{regression_measure} mlr3measures/man/fbeta.Rd0000644000176200001440000000625315170123773015033 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/binary_fbeta.R \name{fbeta} \alias{fbeta} \title{F-beta Score} \usage{ fbeta( truth, response, positive, sample_weights = NULL, beta = 1, na_value = NaN, ... ) } \arguments{ \item{truth}{(\code{factor()})\cr True (observed) labels. Must have the exactly same two levels and the same length as \code{response}.} \item{response}{(\code{factor()})\cr Predicted response labels. Must have the exactly same two levels and the same length as \code{truth}.} \item{positive}{(\verb{character(1))}\cr Name of the positive class.} \item{sample_weights}{(\code{numeric()})\cr Vector of non-negative and finite sample weights. Must have the same length as \code{truth}. The vector gets automatically normalized to sum to one. Defaults to equal sample weights.} \item{beta}{(\code{numeric(1)})\cr Parameter to give either precision or recall more weight. Default is \code{1}, resulting in balanced weights.} \item{na_value}{(\code{numeric(1)})\cr Value that should be returned if the measure is not defined for the input (as described in the note). Default is \code{NaN}.} \item{...}{(\code{any})\cr Additional arguments. Currently ignored.} } \value{ Performance value as \code{numeric(1)}. } \description{ Measure to compare true observed labels with predicted labels in binary classification tasks. } \details{ With \eqn{P} as \code{\link[=precision]{precision()}} and \eqn{R} as \code{\link[=recall]{recall()}}, the F-beta Score is defined as \deqn{ (1 + \beta^2) \frac{P \cdot R}{(\beta^2 P) + R}. }{ (1 + beta^2) * (P*R) / ((beta^2 * P) + R). } It measures the effectiveness of retrieval with respect to a user who attaches \eqn{\beta}{beta} times as much importance to recall as precision. For \eqn{\beta = 1}{beta = 1}, this measure is called "F1" score. This measure is undefined if \link{precision} or \link{recall} is undefined, i.e. TP + FP = 0 or TP + FN = 0. } \section{Meta Information}{ \itemize{ \item Type: \code{"binary"} \item Range: \eqn{[0, 1]}{[0, 1]} \item Minimize: \code{FALSE} \item Required prediction: \code{response} } } \examples{ set.seed(1) lvls = c("a", "b") truth = factor(sample(lvls, 10, replace = TRUE), levels = lvls) response = factor(sample(lvls, 10, replace = TRUE), levels = lvls) fbeta(truth, response, positive = "a") } \references{ Rijsbergen, Van CJ (1979). \emph{Information Retrieval}, 2nd edition. Butterworth-Heinemann, Newton, MA, USA. ISBN 408709294. Goutte C, Gaussier E (2005). \dQuote{A Probabilistic Interpretation of Precision, Recall and F-Score, with Implication for Evaluation.} In \emph{Lecture Notes in Computer Science}, 345--359. \doi{10.1007/978-3-540-31865-1_25}. } \seealso{ Other Binary Classification Measures: \code{\link{auc}()}, \code{\link{bbrier}()}, \code{\link{dor}()}, \code{\link{fdr}()}, \code{\link{fn}()}, \code{\link{fnr}()}, \code{\link{fomr}()}, \code{\link{fp}()}, \code{\link{fpr}()}, \code{\link{gmean}()}, \code{\link{gpr}()}, \code{\link{npv}()}, \code{\link{ppv}()}, \code{\link{prauc}()}, \code{\link{tn}()}, \code{\link{tnr}()}, \code{\link{tp}()}, \code{\link{tpr}()} } \concept{Binary Classification Measures} \concept{binary_classification_measure} mlr3measures/DESCRIPTION0000644000176200001440000000473615170431602014413 0ustar liggesusersPackage: mlr3measures Title: Performance Measures for 'mlr3' Version: 1.3.0 Authors@R: c( person("Michel", "Lang", , "michellang@gmail.com", role = "aut", comment = c(ORCID = "0000-0001-9754-0393")), person("Martin", "Binder", , "mlr.developer@mb706.com", role = "ctb"), person("Marc", "Becker", , "marcbecker@posteo.de", role = c("cre", "aut"), comment = c(ORCID = "0000-0002-8115-0400")), person("Lona", "Koers", , , role = "aut") ) Description: Implements multiple performance measures for supervised learning. Includes over 40 measures for regression and classification. Additionally, meta information about the performance measures can be queried, e.g. what the best and worst possible performances scores are. License: LGPL-3 URL: https://mlr3measures.mlr-org.com, https://github.com/mlr-org/mlr3measures BugReports: https://github.com/mlr-org/mlr3measures/issues Depends: R (>= 3.3.0) Imports: checkmate, mlr3misc, PRROC Suggests: testthat (>= 3.0.0) Config/testthat/edition: 3 Encoding: UTF-8 RoxygenNote: 7.3.3 Collate: 'assertions.R' 'bibentries.R' 'measures.R' 'binary_auc.R' 'binary_bbrier.R' 'binary_dor.R' 'binary_fbeta.R' 'binary_fdr.R' 'binary_fn.R' 'binary_fnr.R' 'binary_fomr.R' 'binary_fp.R' 'binary_fpr.R' 'binary_gmean.R' 'binary_gpr.R' 'binary_npv.R' 'binary_ppv.R' 'binary_prauc.R' 'binary_tn.R' 'binary_tnr.R' 'binary_tp.R' 'binary_tpr.R' 'classif_acc.R' 'classif_auc.R' 'classif_bacc.R' 'classif_ce.R' 'classif_logloss.R' 'classif_mbrier.R' 'classif_mcc.R' 'classif_zero_one.R' 'confusion_matrix.R' 'helper.R' 'regr_ae.R' 'regr_ape.R' 'regr_bias.R' 'regr_ktau.R' 'regr_linex.R' 'regr_mae.R' 'regr_mape.R' 'regr_maxae.R' 'regr_maxse.R' 'regr_medae.R' 'regr_medse.R' 'regr_mse.R' 'regr_msle.R' 'regr_pbias.R' 'regr_pinball.R' 'regr_rae.R' 'regr_rmse.R' 'regr_rmsle.R' 'regr_rrse.R' 'regr_rse.R' 'regr_rsq.R' 'regr_sae.R' 'regr_se.R' 'regr_sle.R' 'regr_smape.R' 'regr_srho.R' 'regr_sse.R' 'roxygen.R' 'similarity_jaccard.R' 'similarity_phi.R' 'zzz.R' NeedsCompilation: no Packaged: 2026-04-17 12:55:06 UTC; marc Author: Michel Lang [aut] (ORCID: ), Martin Binder [ctb], Marc Becker [cre, aut] (ORCID: ), Lona Koers [aut] Maintainer: Marc Becker Repository: CRAN Date/Publication: 2026-04-17 13:20:02 UTC