diff --git a/.github/workflows/test-task-1.yml b/.github/workflows/test-task-1.yml
new file mode 100644
index 00000000..cfab11bc
--- /dev/null
+++ b/.github/workflows/test-task-1.yml
@@ -0,0 +1,43 @@
+# r cmd check workflow of the mlr3 ecosystem v0.1.0
+# https://github.com/mlr-org/actions
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+
+name: mlr3 & mlr3pipelines change
+
+jobs:
+ r-cmd-check:
+ runs-on: ${{ matrix.config.os }}
+
+ name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+ env:
+ GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - {os: ubuntu-latest, r: 'release'}
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: mlr3
+ run: 'echo -e "Remotes:\n mlr-org/mlr3@feat/train-predict,\n mlr-org/mlr3pipelines$fixt/uses_test_task" >> DESCRIPTION'
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: ${{ matrix.config.r }}
+
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ extra-packages: any::rcmdcheck
+ needs: check
+ - uses: r-lib/actions/check-r-package@v2
diff --git a/.github/workflows/test-task-2.yml b/.github/workflows/test-task-2.yml
new file mode 100644
index 00000000..d67df706
--- /dev/null
+++ b/.github/workflows/test-task-2.yml
@@ -0,0 +1,43 @@
+# r cmd check workflow of the mlr3 ecosystem v0.1.0
+# https://github.com/mlr-org/actions
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+
+name: mlr3 & mlr3pipelines change
+
+jobs:
+ r-cmd-check:
+ runs-on: ${{ matrix.config.os }}
+
+ name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+ env:
+ GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - {os: ubuntu-latest, r: 'release'}
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: mlr3
+ run: 'echo -e "Remotes:\n mlr-org/mlr3@feat/train-predict,\n mlr-org/mlr3pipelines$feat/test-rows" >> DESCRIPTION'
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: ${{ matrix.config.r }}
+
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ extra-packages: any::rcmdcheck
+ needs: check
+ - uses: r-lib/actions/check-r-package@v2
diff --git a/DESCRIPTION b/DESCRIPTION
index b62d7ffd..4a3d2338 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -28,7 +28,7 @@ Imports:
checkmate,
data.table,
mlr3misc (>= 0.9.4),
- paradox,
+ paradox (>= 1.0.0),
R6
Suggests:
DiceKriging,
@@ -45,6 +45,8 @@ Suggests:
rmarkdown,
testthat (>= 3.0.0),
xgboost (>= 1.6.0)
+Remotes:
+ mlr-org/mlr3@feat/inner_valid
Config/testthat/edition: 3
Encoding: UTF-8
NeedsCompilation: no
diff --git a/NAMESPACE b/NAMESPACE
index da5b40b1..77707c6d 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -34,6 +34,7 @@ import(paradox)
importFrom(R6,R6Class)
importFrom(mlr3,LearnerClassif)
importFrom(mlr3,LearnerRegr)
+importFrom(mlr3,assert_validate)
importFrom(mlr3,mlr_learners)
importFrom(stats,predict)
importFrom(stats,reformulate)
diff --git a/NEWS.md b/NEWS.md
index 6711d480..d6916746 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,8 @@
# mlr3learners (development version)
+* feat: `LearnerClassifXgboost` and `LearnerRegrXgboost` now support internal tuning and validation.
+ This now also works in conjunction with `mlr3pipelines`.
+
# mlr3learners 0.6.0
* Adaption to new paradox version 1.0.0.
diff --git a/R/LearnerClassifXgboost.R b/R/LearnerClassifXgboost.R
index dc9833d4..4a10db33 100644
--- a/R/LearnerClassifXgboost.R
+++ b/R/LearnerClassifXgboost.R
@@ -32,13 +32,9 @@
#'
#' @section Early stopping:
#' Early stopping can be used to find the optimal number of boosting rounds.
-#' The `early_stopping_set` parameter controls which set is used to monitor the performance.
-#' Set `early_stopping_set = "test"` to monitor the performance of the model on the test set while training.
-#' The test set for early stopping can be set with the `"test"` row role in the [mlr3::Task].
-#' Additionally, the range must be set in which the performance must increase with `early_stopping_rounds` and the maximum number of boosting rounds with `nrounds`.
-#' While resampling, the test set is automatically applied from the [mlr3::Resampling].
-#' Not that using the test set for early stopping can potentially bias the performance scores.
-#' See the section on early stopping in the examples.
+#' The `early_stopping` parameter controls which set is used to monitor the performance.
+#' Set `early_stopping_rounds` to an integer vaulue to monitor the performance of the model on the validation set while training.
+#' For information on how to configure the valdiation set, see the *Validation* section of [`mlr3::Learner`].
#'
#' @templateVar id classif.xgboost
#' @template learner
@@ -55,19 +51,24 @@
#' # Train learner with early stopping on spam data set
#' task = tsk("spam")
#'
-#' # Split task into training and test set
+#' # Split task into training and validation data
#' split = partition(task, ratio = 0.8)
-#' task$set_row_roles(split$test, "test")
+#' task$divide(split$test)
+#'
+#' task
#'
#' # Set early stopping parameter
#' learner = lrn("classif.xgboost",
#' nrounds = 100,
#' early_stopping_rounds = 10,
-#' early_stopping_set = "test"
+#' validate = "internal_valid"
#' )
#'
#' # Train learner with early stopping
#' learner$train(task)
+#'
+#' learner$internal_tuned_values
+#' learner$internal_valid_scores
#' }
LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
inherit = LearnerClassif,
@@ -77,6 +78,16 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
#' @description
#' Creates a new instance of this [R6][R6::R6Class] class.
initialize = function() {
+
+ p_nrounds = p_int(1L,
+ tags = c("train", "hotstart", "internal_tuning"),
+ aggr = crate(function(x) as.integer(ceiling(mean(unlist(x)))), .parent = topenv()),
+ in_tune_fn = crate(function(domain, param_vals) {
+ assert_true(!is.null(param_vals$early_stopping), .var.name = "early stopping rounds is set")
+ assert_integerish(domain$upper, len = 1L, any.missing = FALSE) }, .parent = topenv()),
+ disable_in_tune = list(early_stopping_rounds = NULL)
+ )
+
ps = ps(
alpha = p_dbl(0, default = 0, tags = "train"),
approxcontrib = p_lgl(default = FALSE, tags = "predict"),
@@ -89,7 +100,6 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
device = p_uty(default = "cpu", tags = "train"),
disable_default_eval_metric = p_lgl(default = FALSE, tags = "train"),
early_stopping_rounds = p_int(1L, default = NULL, special_vals = list(NULL), tags = "train"),
- early_stopping_set = p_fct(c("none", "train", "test"), default = "none", tags = "train"),
eta = p_dbl(0, 1, default = 0.3, tags = c("train", "control")),
eval_metric = p_uty(tags = "train"),
feature_selector = p_fct(c("cyclic", "shuffle", "random", "greedy", "thrifty"), default = "cyclic", tags = "train", depends = quote(booster == "gblinear")),
@@ -108,8 +118,8 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
min_child_weight = p_dbl(0, default = 1, tags = c("train", "control")),
missing = p_dbl(default = NA, tags = c("train", "predict"), special_vals = list(NA, NA_real_, NULL)),
monotone_constraints = p_uty(default = 0, tags = c("train", "control"), custom_check = crate(function(x) { checkmate::check_integerish(x, lower = -1, upper = 1, any.missing = FALSE) })), # nolint
+ nrounds = p_nrounds,
normalize_type = p_fct(c("tree", "forest"), default = "tree", tags = "train", depends = quote(booster == "dart")),
- nrounds = p_int(1L, tags = c("train", "hotstart")),
nthread = p_int(1L, default = 1L, tags = c("train", "control", "threads")),
ntreelimit = p_int(1L, default = NULL, special_vals = list(NULL), tags = "predict"),
num_parallel_tree = p_int(1L, default = 1L, tags = c("train", "control")),
@@ -141,18 +151,17 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
verbose = p_int(0L, 2L, default = 1L, tags = "train"),
watchlist = p_uty(default = NULL, tags = "train"),
xgb_model = p_uty(default = NULL, tags = "train")
-
)
# custom defaults
- ps$values = list(nrounds = 1L, nthread = 1L, verbose = 0L, early_stopping_set = "none")
+ ps$values = list(nrounds = 1L, nthread = 1L, verbose = 0L)
super$initialize(
id = "classif.xgboost",
predict_types = c("response", "prob"),
param_set = ps,
feature_types = c("logical", "integer", "numeric"),
- properties = c("weights", "missings", "twoclass", "multiclass", "importance", "hotstart_forward"),
+ properties = c("weights", "missings", "twoclass", "multiclass", "importance", "hotstart_forward", "internal_tuning", "validation"),
packages = c("mlr3learners", "xgboost"),
label = "Extreme Gradient Boosting",
man = "mlr3learners::mlr_learners_classif.xgboost"
@@ -174,8 +183,30 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
set_names(imp$Gain, imp$Feature)
}
),
-
+ active = list(
+ #' @field internal_valid_scores
+ #' The last observation of the validation scores for all metrics.
+ #' Extracted from `model$evaluation_log`
+ internal_valid_scores = function() {
+ self$state$internal_valid_scores
+ },
+ #' @field internal_tuned_values
+ #' Returns the early stopped iterations if `early_stopping_rounds` was set during training.
+ internal_tuned_values = function() {
+ self$state$internal_tuned_values
+ },
+ #' @field validate
+ #' How to construct the internal validation data. This parameter can be either `NULL`,
+ #' a ratio, `"test"`, or `"predefined"`.
+ validate = function(rhs) {
+ if (!missing(rhs)) {
+ private$.validate = assert_validate(rhs)
+ }
+ private$.validate
+ }
+ ),
private = list(
+ .validate = NULL,
.train = function(task) {
pv = self$param_set$get_values(tags = "train")
@@ -217,19 +248,18 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
xgboost::setinfo(data, "weight", task$weights$weight)
}
- if (pv$early_stopping_set != "none") {
- pv$watchlist = c(pv$watchlist, list(train = data))
- }
-
# the last element in the watchlist is used as the early stopping set
- if (pv$early_stopping_set == "test" && !is.null(task$row_roles$test)) {
- test_data = task$data(rows = task$row_roles$test, cols = task$feature_names)
- test_label = nlvls - as.integer(task$truth(rows = task$row_roles$test))
+ internal_valid_task = task$internal_valid_task
+ if (!is.null(pv$early_stopping_rounds) && is.null(internal_valid_task)) {
+ stopf("Learner (%s): Configure field 'validate' to enable early stopping.", self$id)
+ }
+ if (!is.null(internal_valid_task)) {
+ test_data = internal_valid_task$data(cols = internal_valid_task$feature_names)
+ test_label = nlvls - as.integer(internal_valid_task$truth(rows = internal_valid_task$row_roles$test))
test_data = xgboost::xgb.DMatrix(data = as_numeric_matrix(test_data), label = test_label)
pv$watchlist = c(pv$watchlist, list(test = test_data))
}
- pv$early_stopping_set = NULL
invoke(xgboost::xgb.train, data = data, .args = pv)
},
@@ -282,7 +312,6 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
if (!is.null(pars_train$early_stopping_rounds)) {
stop("The parameter `early_stopping_rounds` is set. Early stopping and hotstarting are incompatible.")
}
- pars$early_stopping_set = NULL
# Calculate additional boosting iterations
# niter in model and nrounds in ps should be equal after train and continue
@@ -295,10 +324,29 @@ LearnerClassifXgboost = R6Class("LearnerClassifXgboost",
data = xgboost::xgb.DMatrix(data = as_numeric_matrix(data), label = label)
invoke(xgboost::xgb.train, data = data, xgb_model = model, .args = pars)
+ },
+
+ .extract_internal_tuned_values = function() {
+ if (is.null(self$state$param_vals$early_stopping_rounds)) {
+ return(named_list())
+ }
+ list(nrounds = self$model$niter)
+ },
+
+ .extract_internal_valid_scores = function() {
+ if (is.null(self$model$evaluation_log)) {
+ return(named_list())
+ }
+ as.list(self$model$evaluation_log[
+ get(".N"),
+ set_names(get(".SD"), gsub("^test_", "", colnames(get(".SD",)))),
+ .SDcols = patterns("^test_")
+ ])
}
)
)
+
#' @export
default_values.LearnerClassifXgboost = function(x, search_space, task, ...) { # nolint
special_defaults = list(
diff --git a/R/LearnerRegrXgboost.R b/R/LearnerRegrXgboost.R
index c53cd06b..765917da 100644
--- a/R/LearnerRegrXgboost.R
+++ b/R/LearnerRegrXgboost.R
@@ -31,15 +31,12 @@
#' # Train learner with early stopping on spam data set
#' task = tsk("mtcars")
#'
-#' # Split task into training and test set
-#' split = partition(task, ratio = 0.8)
-#' task$set_row_roles(split$test, "test")
-#'
+#' # use 30 percent for validation
#' # Set early stopping parameter
#' learner = lrn("regr.xgboost",
#' nrounds = 100,
#' early_stopping_rounds = 10,
-#' early_stopping_set = "test"
+#' validate = 0.3
#' )
#'
#' # Train learner with early stopping
@@ -52,6 +49,15 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
#' @description
#' Creates a new instance of this [R6][R6::R6Class] class.
initialize = function() {
+
+ p_nrounds = p_int(1L,
+ tags = c("train", "hotstart", "internal_tuning"),
+ aggr = crate(function(x) as.integer(ceiling(mean(unlist(x)))), .parent = topenv()),
+ in_tune_fn = crate(function(domain, param_vals) {
+ assert_true(!is.null(param_vals$early_stopping), .var.name = "early stopping rounds is set")
+ assert_integerish(domain$upper, len = 1L, any.missing = FALSE) }, .parent = topenv()),
+ disable_in_tune = list(early_stopping_rounds = NULL)
+ )
ps = ps(
alpha = p_dbl(0, default = 0, tags = "train"),
approxcontrib = p_lgl(default = FALSE, tags = "predict"),
@@ -64,7 +70,6 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
device = p_uty(default = "cpu", tags = "train"),
disable_default_eval_metric = p_lgl(default = FALSE, tags = "train"),
early_stopping_rounds = p_int(1L, default = NULL, special_vals = list(NULL), tags = "train"),
- early_stopping_set = p_fct(c("none", "train", "test"), default = "none", tags = "train"),
eta = p_dbl(0, 1, default = 0.3, tags = "train"),
eval_metric = p_uty(default = "rmse", tags = "train"),
feature_selector = p_fct(c("cyclic", "shuffle", "random", "greedy", "thrifty"), default = "cyclic", tags = "train", depends = quote(booster == "gblinear")),
@@ -84,7 +89,7 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
missing = p_dbl(default = NA, tags = c("train", "predict"), special_vals = list(NA, NA_real_, NULL)),
monotone_constraints = p_uty(default = 0, tags = c("train", "control"), custom_check = crate(function(x) { checkmate::check_integerish(x, lower = -1, upper = 1, any.missing = FALSE) })), # nolint
normalize_type = p_fct(c("tree", "forest"), default = "tree", tags = "train", depends = quote(booster == "dart")),
- nrounds = p_int(1L, tags = c("train", "hotstart")),
+ nrounds = p_nrounds,
nthread = p_int(1L, default = 1L, tags = c("train", "threads")),
ntreelimit = p_int(1, default = NULL, special_vals = list(NULL), tags = "predict"),
num_parallel_tree = p_int(1L, default = 1L, tags = "train"),
@@ -120,13 +125,13 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
# param deps
# custom defaults
- ps$values = list(nrounds = 1L, nthread = 1L, verbose = 0L, early_stopping_set = "none")
+ ps$values = list(nrounds = 1L, nthread = 1L, verbose = 0L)
super$initialize(
id = "regr.xgboost",
param_set = ps,
feature_types = c("logical", "integer", "numeric"),
- properties = c("weights", "missings", "importance", "hotstart_forward"),
+ properties = c("weights", "missings", "importance", "hotstart_forward", "internal_tuning", "validation"),
packages = c("mlr3learners", "xgboost"),
label = "Extreme Gradient Boosting",
man = "mlr3learners::mlr_learners_regr.xgboost"
@@ -149,7 +154,31 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
}
),
+ active = list(
+ #' @field internal_valid_scores (named `list()`)\cr
+ #' The last observation of the validation scores for all metrics.
+ #' Extracted from `model$evaluation_log`
+ internal_valid_scores = function() {
+ self$state$internal_valid_scores
+ },
+ #' @field internal_tuned_values (named `list()`)\cr
+ #' Returns the early stopped iterations if `early_stopping_rounds` was set during training.
+ internal_tuned_values = function() {
+ self$state$internal_tuned_values
+ },
+ #' @field validate
+ #' How to construct the internal validation data. This parameter can be either `NULL`,
+ #' a ratio, `"test"`, or `"internal_valid"`.
+ validate = function(rhs) {
+ if (!missing(rhs)) {
+ private$.validate = assert_validate(rhs)
+ }
+ private$.validate
+ }
+ ),
+
private = list(
+ .validate = NULL,
.train = function(task) {
pv = self$param_set$get_values(tags = "train")
@@ -166,19 +195,18 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
xgboost::setinfo(data, "weight", task$weights$weight)
}
- if (pv$early_stopping_set != "none") {
- pv$watchlist = c(pv$watchlist, list(train = data))
- }
-
# the last element in the watchlist is used as the early stopping set
- if (pv$early_stopping_set == "test" && !is.null(task$row_roles$test)) {
+ internal_valid_task = task$internal_valid_task
+ if (!is.null(pv$early_stopping_rounds) && is.null(internal_valid_task)) {
+ stopf("Learner (%s): Configure field 'validate' to enable early stopping.", self$id)
+ }
+ if (!is.null(internal_valid_task)) {
test_data = task$data(rows = task$row_roles$test, cols = task$feature_names)
test_target = task$data(rows = task$row_roles$test, cols = task$target_names)
test_data = xgboost::xgb.DMatrix(data = as_numeric_matrix(test_data), label = data.matrix(test_target))
pv$watchlist = c(pv$watchlist, list(test = test_data))
}
- pv$early_stopping_set = NULL
invoke(xgboost::xgb.train, data = data, .args = pv)
},
@@ -199,7 +227,6 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
if (!is.null(pars_train$early_stopping_rounds)) {
stop("The parameter `early_stopping_rounds` is set. Early stopping and hotstarting are incompatible.")
}
- pars$early_stopping_set = NULL
# Calculate additional boosting iterations
# niter in model and nrounds in ps should be equal after train and continue
@@ -211,6 +238,24 @@ LearnerRegrXgboost = R6Class("LearnerRegrXgboost",
data = xgboost::xgb.DMatrix(data = as_numeric_matrix(data), label = data.matrix(target))
invoke(xgboost::xgb.train, data = data, xgb_model = model, .args = pars)
+ },
+
+ .extract_internal_tuned_values = function() {
+ if (is.null(self$state$param_vals$early_stopping_rounds)) {
+ return(named_list())
+ }
+ list(nrounds = self$model$niter)
+ },
+
+ .extract_internal_valid_scores = function() {
+ if (is.null(self$model$evaluation_log)) {
+ return(named_list())
+ }
+ as.list(self$model$evaluation_log[
+ get(".N"),
+ set_names(get(".SD"), gsub("^test_", "", colnames(get(".SD",)))),
+ .SDcols = patterns("^test_")
+ ])
}
)
)
diff --git a/R/zzz.R b/R/zzz.R
index e3074d18..533f5650 100644
--- a/R/zzz.R
+++ b/R/zzz.R
@@ -3,7 +3,7 @@
#' @import mlr3misc
#' @import checkmate
#' @importFrom R6 R6Class
-#' @importFrom mlr3 mlr_learners LearnerClassif LearnerRegr
+#' @importFrom mlr3 mlr_learners LearnerClassif LearnerRegr assert_validate
#' @importFrom stats predict reformulate
#'
#' @description
@@ -13,6 +13,8 @@
#' Feel invited to contribute a missing learner to the \CRANpkg{mlr3} ecosystem!
"_PACKAGE"
+utils::globalVariables(c("patterns"))
+
#' @include aaa.R
register_mlr3 = function() {
x = utils::getFromNamespace("mlr_learners", ns = "mlr3")
diff --git a/inst/paramtest/test_paramtest_classif.xgboost.R b/inst/paramtest/test_paramtest_classif.xgboost.R
index 7aa7ef7f..d00c721b 100644
--- a/inst/paramtest/test_paramtest_classif.xgboost.R
+++ b/inst/paramtest/test_paramtest_classif.xgboost.R
@@ -2,10 +2,12 @@ library(mlr3learners)
library(magrittr, exclude = c("equals", "is_less_than", "not"))
library(rvest)
-add_params_xgboost = read_html("https://xgboost.readthedocs.io/en/latest/parameter.html") %>%
- html_elements("li") %>%
- html_elements("p") %>%
- html_text2() %>%
+x = rvest::read_html("https://xgboost.readthedocs.io/en/latest/parameter.html")
+xli = rvest::html_elements(x, "li")
+xp = rvest::html_elements(x, "p")
+x = c(rvest::html_text2(xli), rvest::html_text2(xp))
+
+add_params_xgboost = x %>%
grep("default=", ., value = T) %>%
strsplit(., split = " ") %>%
mlr3misc::map_chr(., function(x) x[1]) %>%
@@ -40,7 +42,7 @@ test_that("classif.xgboost", {
"label", # handled by mlr3
"weight", # handled by mlr3
"nthread", # handled by mlr3
- "early_stopping_set" # extra parameter of mlr3
+ "early_stopping" # extra parameter of mlr3
)
ParamTest = run_paramtest(learner, fun, exclude, tag = "train")
diff --git a/inst/paramtest/test_paramtest_regr.xgboost.R b/inst/paramtest/test_paramtest_regr.xgboost.R
index 1b1d1fd4..ea7f54d4 100644
--- a/inst/paramtest/test_paramtest_regr.xgboost.R
+++ b/inst/paramtest/test_paramtest_regr.xgboost.R
@@ -2,10 +2,12 @@ library(mlr3learners)
library(magrittr, exclude = c("equals", "is_less_than", "not"))
library(rvest)
-add_params_xgboost = read_html("https://xgboost.readthedocs.io/en/latest/parameter.html") %>%
- html_elements("li") %>%
- html_elements("p") %>%
- html_text2() %>%
+x = rvest::read_html("https://xgboost.readthedocs.io/en/latest/parameter.html")
+xli = rvest::html_elements(x, "li")
+xp = rvest::html_elements(x, "p")
+x = c(rvest::html_text2(xli), rvest::html_text2(xp))
+
+add_params_xgboost = x %>%
grep("default=", ., value = T) %>%
strsplit(., split = " ") %>%
mlr3misc::map_chr(., function(x) x[1]) %>%
@@ -39,8 +41,8 @@ test_that("regr.xgboost", {
"eval_metric", # handled by mlr3
"label", # handled by mlr3
"weight", # handled by mlr3
- "nthread", # handled by mlr3
- "early_stopping_set" # extra parameter of mlr3
+ "nthread" # handled by mlr3
+ "early_stopping" # extra parameter of mlr3
)
ParamTest = run_paramtest(learner, fun, exclude, tag = "train")
diff --git a/man-roxygen/example_early_stopping.R b/man-roxygen/example_early_stopping.R
index a05530e6..f16a7cf1 100644
--- a/man-roxygen/example_early_stopping.R
+++ b/man-roxygen/example_early_stopping.R
@@ -5,13 +5,13 @@
#'
#' # Split task into training and test set
#' split = partition(task, ratio = 0.8)
-#' task$set_row_roles(split$test, "test")
+#' task$set_row_roles(split$test, "validation")
#'
#' # Set early stopping parameter
#' learner = lrn("<%= id %>",
#' nrounds = 1000,
#' early_stopping_rounds = 100,
-#' early_stopping_set = "test"
+#' early_stopping = TRUE
#' )
#'
#' # Train learner with early stopping
diff --git a/man/mlr_learners_classif.cv_glmnet.Rd b/man/mlr_learners_classif.cv_glmnet.Rd
index b298432e..03d744ba 100644
--- a/man/mlr_learners_classif.cv_glmnet.Rd
+++ b/man/mlr_learners_classif.cv_glmnet.Rd
@@ -43,7 +43,7 @@ lrn("classif.cv_glmnet")
exclude \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
exmx \tab numeric \tab 250 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
fdev \tab numeric \tab 1e-05 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
- foldid \tab untyped \tab \tab \tab - \cr
+ foldid \tab untyped \tab NULL \tab \tab - \cr
gamma \tab untyped \tab - \tab \tab - \cr
grouped \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
intercept \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
@@ -57,7 +57,7 @@ lrn("classif.cv_glmnet")
mxit \tab integer \tab 100 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
nfolds \tab integer \tab 10 \tab \tab \eqn{[3, \infty)}{[3, Inf)} \cr
nlambda \tab integer \tab 100 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
- offset \tab untyped \tab \tab \tab - \cr
+ offset \tab untyped \tab NULL \tab \tab - \cr
parallel \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
penalty.factor \tab untyped \tab - \tab \tab - \cr
pmax \tab integer \tab - \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
diff --git a/man/mlr_learners_classif.glmnet.Rd b/man/mlr_learners_classif.glmnet.Rd
index 7cb1b4be..44086b9e 100644
--- a/man/mlr_learners_classif.glmnet.Rd
+++ b/man/mlr_learners_classif.glmnet.Rd
@@ -69,7 +69,7 @@ lrn("classif.glmnet")
mxitnr \tab integer \tab 25 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
nlambda \tab integer \tab 100 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
newoffset \tab untyped \tab - \tab \tab - \cr
- offset \tab untyped \tab \tab \tab - \cr
+ offset \tab untyped \tab NULL \tab \tab - \cr
penalty.factor \tab untyped \tab - \tab \tab - \cr
pmax \tab integer \tab - \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
pmin \tab numeric \tab 1e-09 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
diff --git a/man/mlr_learners_classif.kknn.Rd b/man/mlr_learners_classif.kknn.Rd
index 7f386b5d..304cc7dd 100644
--- a/man/mlr_learners_classif.kknn.Rd
+++ b/man/mlr_learners_classif.kknn.Rd
@@ -56,7 +56,7 @@ lrn("classif.kknn")
distance \tab numeric \tab 2 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
kernel \tab character \tab optimal \tab rectangular, triangular, epanechnikov, biweight, triweight, cos, inv, gaussian, rank, optimal \tab - \cr
scale \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
- ykernel \tab untyped \tab \tab \tab - \cr
+ ykernel \tab untyped \tab NULL \tab \tab - \cr
store_model \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
}
}
diff --git a/man/mlr_learners_classif.log_reg.Rd b/man/mlr_learners_classif.log_reg.Rd
index 16029543..f85f2389 100644
--- a/man/mlr_learners_classif.log_reg.Rd
+++ b/man/mlr_learners_classif.log_reg.Rd
@@ -55,7 +55,7 @@ lrn("classif.log_reg")
\section{Parameters}{
\tabular{lllll}{
Id \tab Type \tab Default \tab Levels \tab Range \cr
- dispersion \tab untyped \tab \tab \tab - \cr
+ dispersion \tab untyped \tab NULL \tab \tab - \cr
epsilon \tab numeric \tab 1e-08 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
etastart \tab untyped \tab - \tab \tab - \cr
maxit \tab numeric \tab 25 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
@@ -63,7 +63,7 @@ lrn("classif.log_reg")
mustart \tab untyped \tab - \tab \tab - \cr
offset \tab untyped \tab - \tab \tab - \cr
singular.ok \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
- start \tab untyped \tab \tab \tab - \cr
+ start \tab untyped \tab NULL \tab \tab - \cr
trace \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
x \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
y \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
diff --git a/man/mlr_learners_classif.nnet.Rd b/man/mlr_learners_classif.nnet.Rd
index 435f44ca..66593366 100644
--- a/man/mlr_learners_classif.nnet.Rd
+++ b/man/mlr_learners_classif.nnet.Rd
@@ -38,7 +38,7 @@ lrn("classif.nnet")
Wts \tab untyped \tab - \tab \tab - \cr
abstol \tab numeric \tab 1e-04 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
censored \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
- contrasts \tab untyped \tab \tab \tab - \cr
+ contrasts \tab untyped \tab NULL \tab \tab - \cr
decay \tab numeric \tab 0 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
mask \tab untyped \tab - \tab \tab - \cr
maxit \tab integer \tab 100 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
diff --git a/man/mlr_learners_classif.ranger.Rd b/man/mlr_learners_classif.ranger.Rd
index 6cd38f2f..a0681877 100644
--- a/man/mlr_learners_classif.ranger.Rd
+++ b/man/mlr_learners_classif.ranger.Rd
@@ -56,7 +56,7 @@ lrn("classif.ranger")
Id \tab Type \tab Default \tab Levels \tab Range \cr
alpha \tab numeric \tab 0.5 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
always.split.variables \tab untyped \tab - \tab \tab - \cr
- class.weights \tab untyped \tab \tab \tab - \cr
+ class.weights \tab untyped \tab NULL \tab \tab - \cr
holdout \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
importance \tab character \tab - \tab none, impurity, impurity_corrected, permutation \tab - \cr
keep.inbag \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
@@ -80,7 +80,7 @@ lrn("classif.ranger")
scale.permutation.importance \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
se.method \tab character \tab infjack \tab jack, infjack \tab - \cr
seed \tab integer \tab NULL \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
- split.select.weights \tab untyped \tab \tab \tab - \cr
+ split.select.weights \tab untyped \tab NULL \tab \tab - \cr
splitrule \tab character \tab gini \tab gini, extratrees, hellinger \tab - \cr
verbose \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
write.forest \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
diff --git a/man/mlr_learners_classif.svm.Rd b/man/mlr_learners_classif.svm.Rd
index cd817879..659376f3 100644
--- a/man/mlr_learners_classif.svm.Rd
+++ b/man/mlr_learners_classif.svm.Rd
@@ -31,7 +31,7 @@ lrn("classif.svm")
\tabular{lllll}{
Id \tab Type \tab Default \tab Levels \tab Range \cr
cachesize \tab numeric \tab 40 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
- class.weights \tab untyped \tab \tab \tab - \cr
+ class.weights \tab untyped \tab NULL \tab \tab - \cr
coef0 \tab numeric \tab 0 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
cost \tab numeric \tab 1 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
cross \tab integer \tab 0 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
diff --git a/man/mlr_learners_classif.xgboost.Rd b/man/mlr_learners_classif.xgboost.Rd
index ca5f8132..d8186a99 100644
--- a/man/mlr_learners_classif.xgboost.Rd
+++ b/man/mlr_learners_classif.xgboost.Rd
@@ -49,13 +49,9 @@ would error. Just setting a nonsense default to workaround this.
\section{Early stopping}{
Early stopping can be used to find the optimal number of boosting rounds.
-The \code{early_stopping_set} parameter controls which set is used to monitor the performance.
-Set \code{early_stopping_set = "test"} to monitor the performance of the model on the test set while training.
-The test set for early stopping can be set with the \code{"test"} row role in the \link[mlr3:Task]{mlr3::Task}.
-Additionally, the range must be set in which the performance must increase with \code{early_stopping_rounds} and the maximum number of boosting rounds with \code{nrounds}.
-While resampling, the test set is automatically applied from the \link[mlr3:Resampling]{mlr3::Resampling}.
-Not that using the test set for early stopping can potentially bias the performance scores.
-See the section on early stopping in the examples.
+The \code{early_stopping} parameter controls which set is used to monitor the performance.
+Set \code{early_stopping_rounds} to an integer vaulue to monitor the performance of the model on the validation set while training.
+For information on how to configure the valdiation set, see the \emph{Validation} section of \code{\link[mlr3:Learner]{mlr3::Learner}}.
}
\section{Dictionary}{
@@ -84,18 +80,17 @@ lrn("classif.xgboost")
approxcontrib \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
base_score \tab numeric \tab 0.5 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
booster \tab character \tab gbtree \tab gbtree, gblinear, dart \tab - \cr
- callbacks \tab untyped \tab list \tab \tab - \cr
+ callbacks \tab untyped \tab list() \tab \tab - \cr
colsample_bylevel \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
colsample_bynode \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
colsample_bytree \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
- device \tab untyped \tab cpu \tab \tab - \cr
+ device \tab untyped \tab "cpu" \tab \tab - \cr
disable_default_eval_metric \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
early_stopping_rounds \tab integer \tab NULL \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
- early_stopping_set \tab character \tab none \tab none, train, test \tab - \cr
eta \tab numeric \tab 0.3 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
eval_metric \tab untyped \tab - \tab \tab - \cr
feature_selector \tab character \tab cyclic \tab cyclic, shuffle, random, greedy, thrifty \tab - \cr
- feval \tab untyped \tab \tab \tab - \cr
+ feval \tab untyped \tab NULL \tab \tab - \cr
gamma \tab numeric \tab 0 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
grow_policy \tab character \tab depthwise \tab depthwise, lossguide \tab - \cr
interaction_constraints \tab untyped \tab - \tab \tab - \cr
@@ -110,12 +105,12 @@ lrn("classif.xgboost")
min_child_weight \tab numeric \tab 1 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
missing \tab numeric \tab NA \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
monotone_constraints \tab untyped \tab 0 \tab \tab - \cr
- normalize_type \tab character \tab tree \tab tree, forest \tab - \cr
nrounds \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
+ normalize_type \tab character \tab tree \tab tree, forest \tab - \cr
nthread \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
ntreelimit \tab integer \tab NULL \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
num_parallel_tree \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
- objective \tab untyped \tab binary:logistic \tab \tab - \cr
+ objective \tab untyped \tab "binary:logistic" \tab \tab - \cr
one_drop \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
outputmargin \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
predcontrib \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
@@ -129,7 +124,7 @@ lrn("classif.xgboost")
seed_per_iteration \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
sampling_method \tab character \tab uniform \tab uniform, gradient_based \tab - \cr
sample_type \tab character \tab uniform \tab uniform, weighted \tab - \cr
- save_name \tab untyped \tab \tab \tab - \cr
+ save_name \tab untyped \tab NULL \tab \tab - \cr
save_period \tab integer \tab NULL \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
scale_pos_weight \tab numeric \tab 1 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
skip_drop \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
@@ -141,8 +136,8 @@ lrn("classif.xgboost")
tweedie_variance_power \tab numeric \tab 1.5 \tab \tab \eqn{[1, 2]}{[1, 2]} \cr
updater \tab untyped \tab - \tab \tab - \cr
verbose \tab integer \tab 1 \tab \tab \eqn{[0, 2]}{[0, 2]} \cr
- watchlist \tab untyped \tab \tab \tab - \cr
- xgb_model \tab untyped \tab \tab \tab - \cr
+ watchlist \tab untyped \tab NULL \tab \tab - \cr
+ xgb_model \tab untyped \tab NULL \tab \tab - \cr
}
}
@@ -180,19 +175,24 @@ predictions$score()
# Train learner with early stopping on spam data set
task = tsk("spam")
-# Split task into training and test set
+# Split task into training and validation data
split = partition(task, ratio = 0.8)
-task$set_row_roles(split$test, "test")
+task$divide(split$test)
+
+task
# Set early stopping parameter
learner = lrn("classif.xgboost",
nrounds = 100,
early_stopping_rounds = 10,
- early_stopping_set = "test"
+ validate = "internal_valid"
)
# Train learner with early stopping
learner$train(task)
+
+learner$internal_tuned_values
+learner$internal_valid_scores
}
}
\references{
@@ -245,6 +245,19 @@ Other Learner:
\section{Super classes}{
\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerClassif]{mlr3::LearnerClassif}} -> \code{LearnerClassifXgboost}
}
+\section{Active bindings}{
+\if{html}{\out{}}
+\describe{
+\item{\code{internal_valid_scores}}{The last observation of the validation scores for all metrics.
+Extracted from \code{model$evaluation_log}}
+
+\item{\code{internal_tuned_values}}{Returns the early stopped iterations if \code{early_stopping_rounds} was set during training.}
+
+\item{\code{validate}}{How to construct the internal validation data. This parameter can be either \code{NULL},
+a ratio, \code{"test"}, or \code{"predefined"}.}
+}
+\if{html}{\out{
}}
+}
\section{Methods}{
\subsection{Public methods}{
\itemize{
diff --git a/man/mlr_learners_regr.cv_glmnet.Rd b/man/mlr_learners_regr.cv_glmnet.Rd
index 1ab38588..0b68ac51 100644
--- a/man/mlr_learners_regr.cv_glmnet.Rd
+++ b/man/mlr_learners_regr.cv_glmnet.Rd
@@ -43,7 +43,7 @@ lrn("regr.cv_glmnet")
exmx \tab numeric \tab 250 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
family \tab character \tab gaussian \tab gaussian, poisson \tab - \cr
fdev \tab numeric \tab 1e-05 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
- foldid \tab untyped \tab \tab \tab - \cr
+ foldid \tab untyped \tab NULL \tab \tab - \cr
gamma \tab untyped \tab - \tab \tab - \cr
grouped \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
intercept \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
@@ -57,7 +57,7 @@ lrn("regr.cv_glmnet")
mxitnr \tab integer \tab 25 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
nfolds \tab integer \tab 10 \tab \tab \eqn{[3, \infty)}{[3, Inf)} \cr
nlambda \tab integer \tab 100 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
- offset \tab untyped \tab \tab \tab - \cr
+ offset \tab untyped \tab NULL \tab \tab - \cr
parallel \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
penalty.factor \tab untyped \tab - \tab \tab - \cr
pmax \tab integer \tab - \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
diff --git a/man/mlr_learners_regr.glmnet.Rd b/man/mlr_learners_regr.glmnet.Rd
index 2938319e..b75062ed 100644
--- a/man/mlr_learners_regr.glmnet.Rd
+++ b/man/mlr_learners_regr.glmnet.Rd
@@ -75,7 +75,7 @@ lrn("regr.glmnet")
mxitnr \tab integer \tab 25 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
newoffset \tab untyped \tab - \tab \tab - \cr
nlambda \tab integer \tab 100 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
- offset \tab untyped \tab \tab \tab - \cr
+ offset \tab untyped \tab NULL \tab \tab - \cr
parallel \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
penalty.factor \tab untyped \tab - \tab \tab - \cr
pmax \tab integer \tab - \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
diff --git a/man/mlr_learners_regr.kknn.Rd b/man/mlr_learners_regr.kknn.Rd
index 48346ecb..39fe7fda 100644
--- a/man/mlr_learners_regr.kknn.Rd
+++ b/man/mlr_learners_regr.kknn.Rd
@@ -56,7 +56,7 @@ lrn("regr.kknn")
distance \tab numeric \tab 2 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
kernel \tab character \tab optimal \tab rectangular, triangular, epanechnikov, biweight, triweight, cos, inv, gaussian, rank, optimal \tab - \cr
scale \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
- ykernel \tab untyped \tab \tab \tab - \cr
+ ykernel \tab untyped \tab NULL \tab \tab - \cr
store_model \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
}
}
diff --git a/man/mlr_learners_regr.km.Rd b/man/mlr_learners_regr.km.Rd
index 79ac415f..387fe8f8 100644
--- a/man/mlr_learners_regr.km.Rd
+++ b/man/mlr_learners_regr.km.Rd
@@ -41,32 +41,32 @@ lrn("regr.km")
Id \tab Type \tab Default \tab Levels \tab Range \cr
bias.correct \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
checkNames \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
- coef.cov \tab untyped \tab \tab \tab - \cr
- coef.trend \tab untyped \tab \tab \tab - \cr
- coef.var \tab untyped \tab \tab \tab - \cr
- control \tab untyped \tab \tab \tab - \cr
+ coef.cov \tab untyped \tab NULL \tab \tab - \cr
+ coef.trend \tab untyped \tab NULL \tab \tab - \cr
+ coef.var \tab untyped \tab NULL \tab \tab - \cr
+ control \tab untyped \tab NULL \tab \tab - \cr
cov.compute \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
covtype \tab character \tab matern5_2 \tab gauss, matern5_2, matern3_2, exp, powexp \tab - \cr
estim.method \tab character \tab MLE \tab MLE, LOO \tab - \cr
gr \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
iso \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
jitter \tab numeric \tab 0 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
- kernel \tab untyped \tab \tab \tab - \cr
- knots \tab untyped \tab \tab \tab - \cr
+ kernel \tab untyped \tab NULL \tab \tab - \cr
+ knots \tab untyped \tab NULL \tab \tab - \cr
light.return \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
- lower \tab untyped \tab \tab \tab - \cr
+ lower \tab untyped \tab NULL \tab \tab - \cr
multistart \tab integer \tab 1 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
- noise.var \tab untyped \tab \tab \tab - \cr
+ noise.var \tab untyped \tab NULL \tab \tab - \cr
nugget \tab numeric \tab - \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
nugget.estim \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
nugget.stability \tab numeric \tab 0 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
optim.method \tab character \tab BFGS \tab BFGS, gen \tab - \cr
- parinit \tab untyped \tab \tab \tab - \cr
- penalty \tab untyped \tab \tab \tab - \cr
+ parinit \tab untyped \tab NULL \tab \tab - \cr
+ penalty \tab untyped \tab NULL \tab \tab - \cr
scaling \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
se.compute \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
type \tab character \tab SK \tab SK, UK \tab - \cr
- upper \tab untyped \tab \tab \tab - \cr
+ upper \tab untyped \tab NULL \tab \tab - \cr
}
}
diff --git a/man/mlr_learners_regr.nnet.Rd b/man/mlr_learners_regr.nnet.Rd
index 6479a702..3fcf40f9 100644
--- a/man/mlr_learners_regr.nnet.Rd
+++ b/man/mlr_learners_regr.nnet.Rd
@@ -38,7 +38,7 @@ lrn("regr.nnet")
Wts \tab untyped \tab - \tab \tab - \cr
abstol \tab numeric \tab 1e-04 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
censored \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
- contrasts \tab untyped \tab \tab \tab - \cr
+ contrasts \tab untyped \tab NULL \tab \tab - \cr
decay \tab numeric \tab 0 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
mask \tab untyped \tab - \tab \tab - \cr
maxit \tab integer \tab 100 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
diff --git a/man/mlr_learners_regr.ranger.Rd b/man/mlr_learners_regr.ranger.Rd
index f980e806..b4af09f4 100644
--- a/man/mlr_learners_regr.ranger.Rd
+++ b/man/mlr_learners_regr.ranger.Rd
@@ -56,7 +56,7 @@ lrn("regr.ranger")
scale.permutation.importance \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
se.method \tab character \tab infjack \tab jack, infjack \tab - \cr
seed \tab integer \tab NULL \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
- split.select.weights \tab untyped \tab \tab \tab - \cr
+ split.select.weights \tab untyped \tab NULL \tab \tab - \cr
splitrule \tab character \tab variance \tab variance, extratrees, maxstat \tab - \cr
verbose \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
write.forest \tab logical \tab TRUE \tab TRUE, FALSE \tab - \cr
diff --git a/man/mlr_learners_regr.xgboost.Rd b/man/mlr_learners_regr.xgboost.Rd
index 386fb73d..8148b479 100644
--- a/man/mlr_learners_regr.xgboost.Rd
+++ b/man/mlr_learners_regr.xgboost.Rd
@@ -45,18 +45,17 @@ lrn("regr.xgboost")
approxcontrib \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
base_score \tab numeric \tab 0.5 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
booster \tab character \tab gbtree \tab gbtree, gblinear, dart \tab - \cr
- callbacks \tab untyped \tab list \tab \tab - \cr
+ callbacks \tab untyped \tab list() \tab \tab - \cr
colsample_bylevel \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
colsample_bynode \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
colsample_bytree \tab numeric \tab 1 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
- device \tab untyped \tab cpu \tab \tab - \cr
+ device \tab untyped \tab "cpu" \tab \tab - \cr
disable_default_eval_metric \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
early_stopping_rounds \tab integer \tab NULL \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
- early_stopping_set \tab character \tab none \tab none, train, test \tab - \cr
eta \tab numeric \tab 0.3 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr
- eval_metric \tab untyped \tab rmse \tab \tab - \cr
+ eval_metric \tab untyped \tab "rmse" \tab \tab - \cr
feature_selector \tab character \tab cyclic \tab cyclic, shuffle, random, greedy, thrifty \tab - \cr
- feval \tab untyped \tab \tab \tab - \cr
+ feval \tab untyped \tab NULL \tab \tab - \cr
gamma \tab numeric \tab 0 \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
grow_policy \tab character \tab depthwise \tab depthwise, lossguide \tab - \cr
interaction_constraints \tab untyped \tab - \tab \tab - \cr
@@ -76,7 +75,7 @@ lrn("regr.xgboost")
nthread \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
ntreelimit \tab integer \tab NULL \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
num_parallel_tree \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr
- objective \tab untyped \tab reg:squarederror \tab \tab - \cr
+ objective \tab untyped \tab "reg:squarederror" \tab \tab - \cr
one_drop \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
outputmargin \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
predcontrib \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
@@ -89,7 +88,7 @@ lrn("regr.xgboost")
reshape \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
sampling_method \tab character \tab uniform \tab uniform, gradient_based \tab - \cr
sample_type \tab character \tab uniform \tab uniform, weighted \tab - \cr
- save_name \tab untyped \tab \tab \tab - \cr
+ save_name \tab untyped \tab NULL \tab \tab - \cr
save_period \tab integer \tab NULL \tab \tab \eqn{[0, \infty)}{[0, Inf)} \cr
scale_pos_weight \tab numeric \tab 1 \tab \tab \eqn{(-\infty, \infty)}{(-Inf, Inf)} \cr
seed_per_iteration \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr
@@ -102,21 +101,17 @@ lrn("regr.xgboost")
tweedie_variance_power \tab numeric \tab 1.5 \tab \tab \eqn{[1, 2]}{[1, 2]} \cr
updater \tab untyped \tab - \tab \tab - \cr
verbose \tab integer \tab 1 \tab \tab \eqn{[0, 2]}{[0, 2]} \cr
- watchlist \tab untyped \tab \tab \tab - \cr
- xgb_model \tab untyped \tab \tab \tab - \cr
+ watchlist \tab untyped \tab NULL \tab \tab - \cr
+ xgb_model \tab untyped \tab NULL \tab \tab - \cr
}
}
\section{Early stopping}{
Early stopping can be used to find the optimal number of boosting rounds.
-The \code{early_stopping_set} parameter controls which set is used to monitor the performance.
-Set \code{early_stopping_set = "test"} to monitor the performance of the model on the test set while training.
-The test set for early stopping can be set with the \code{"test"} row role in the \link[mlr3:Task]{mlr3::Task}.
-Additionally, the range must be set in which the performance must increase with \code{early_stopping_rounds} and the maximum number of boosting rounds with \code{nrounds}.
-While resampling, the test set is automatically applied from the \link[mlr3:Resampling]{mlr3::Resampling}.
-Not that using the test set for early stopping can potentially bias the performance scores.
-See the section on early stopping in the examples.
+The \code{early_stopping} parameter controls which set is used to monitor the performance.
+Set \code{early_stopping_rounds} to an integer vaulue to monitor the performance of the model on the validation set while training.
+For information on how to configure the valdiation set, see the \emph{Validation} section of \code{\link[mlr3:Learner]{mlr3::Learner}}.
}
\section{Initial parameter values}{
@@ -179,15 +174,12 @@ predictions$score()
# Train learner with early stopping on spam data set
task = tsk("mtcars")
-# Split task into training and test set
-split = partition(task, ratio = 0.8)
-task$set_row_roles(split$test, "test")
-
+# use 30 percent for validation
# Set early stopping parameter
learner = lrn("regr.xgboost",
nrounds = 100,
early_stopping_rounds = 10,
- early_stopping_set = "test"
+ validate = 0.3
)
# Train learner with early stopping
@@ -244,6 +236,21 @@ Other Learner:
\section{Super classes}{
\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrXgboost}
}
+\section{Active bindings}{
+\if{html}{\out{}}
+\describe{
+\item{\code{internal_valid_scores}}{(named \code{list()})\cr
+The last observation of the validation scores for all metrics.
+Extracted from \code{model$evaluation_log}}
+
+\item{\code{internal_tuned_values}}{(named \code{list()})\cr
+Returns the early stopped iterations if \code{early_stopping_rounds} was set during training.}
+
+\item{\code{validate}}{How to construct the internal validation data. This parameter can be either \code{NULL},
+a ratio, \code{"test"}, or \code{"internal_valid"}.}
+}
+\if{html}{\out{
}}
+}
\section{Methods}{
\subsection{Public methods}{
\itemize{
diff --git a/tests/testthat/test_classif_xgboost.R b/tests/testthat/test_classif_xgboost.R
index 7954853d..9739e76e 100644
--- a/tests/testthat/test_classif_xgboost.R
+++ b/tests/testthat/test_classif_xgboost.R
@@ -63,16 +63,45 @@ test_that("hotstart", {
expect_equal(learner_4$state$param_vals$nrounds, 5L)
})
-test_that("early stopping on the test set works", {
+test_that("validation and inner tuning", {
task = tsk("spam")
- split = partition(task, ratio = 0.8)
- task$set_row_roles(split$test, "test")
+
learner = lrn("classif.xgboost",
- nrounds = 1000,
- early_stopping_rounds = 100,
- early_stopping_set = "test"
+ nrounds = 10,
+ early_stopping_rounds = 1,
+ validate = 0.2
)
learner$train(task)
- expect_named(learner$model$evaluation_log, c("iter", "train_logloss", "test_logloss"))
+ expect_named(learner$model$evaluation_log, c("iter", "test_logloss"))
+ expect_list(learner$internal_valid_scores, types = "numeric")
+ expect_equal(names(learner$internal_valid_scores), "logloss")
+ expect_equal(learner$internal_valid_scores$logloss, learner$model$evaluation[get("iter") == 10, "test_logloss"][[1L]])
+
+ expect_list(learner$internal_tuned_values, types = "integerish")
+ expect_equal(names(learner$internal_tuned_values), "nrounds")
+
+ learner$validate = NULL
+ expect_error(learner$train(task), "field 'validate'")
+
+ learner$validate = 0.2
+ task$internal_valid_task = NULL
+ learner$param_set$set_values(
+ early_stopping_rounds = NULL
+ )
+ learner$train(task)
+ expect_equal(learner$internal_tuned_values, named_list())
+ expect_named(learner$model$evaluation_log, c("iter", "test_logloss"))
+ expect_list(learner$internal_valid_scores, types = "numeric")
+ expect_equal(names(learner$internal_valid_scores), "logloss")
+
+ learner = lrn("classif.xgboost",
+ nrounds = to_tune(upper = 1000, internal = TRUE),
+ validate = 0.2
+ )
+ s = learner$param_set$search_space()
+ expect_error(learner$param_set$convert_internal_search_space(s), "early stopping")
+ learner$param_set$set_values(early_stopping_rounds = 10)
+ learner$param_set$disable_internal_tuning("nrounds")
+ expect_equal(learner$param_set$values$early_stopping_rounds, NULL)
})
diff --git a/tests/testthat/test_regr_xgboost.R b/tests/testthat/test_regr_xgboost.R
index 1f1ba0d6..9b0661d2 100644
--- a/tests/testthat/test_regr_xgboost.R
+++ b/tests/testthat/test_regr_xgboost.R
@@ -44,16 +44,45 @@ test_that("hotstart", {
expect_equal(learner_4$state$param_vals$nrounds, 5L)
})
-test_that("early stopping on the test set works", {
+test_that("validation and inner tuning", {
task = tsk("mtcars")
- split = partition(task, ratio = 0.8)
- task$set_row_roles(split$test, "test")
+
learner = lrn("regr.xgboost",
- nrounds = 1000,
- early_stopping_rounds = 100,
- early_stopping_set = "test"
+ nrounds = 10,
+ early_stopping_rounds = 1,
+ validate = 0.2
)
learner$train(task)
- expect_named(learner$model$evaluation_log, c("iter", "train_rmse", "test_rmse"))
+ expect_named(learner$model$evaluation_log, c("iter", "test_rmse"))
+ expect_list(learner$internal_valid_scores, types = "numeric")
+ expect_equal(names(learner$internal_valid_scores), "rmse")
+ expect_equal(learner$internal_valid_scores$rmse, learner$model$evaluation[get("iter") == 10, "test_rmse"][[1L]])
+
+ expect_list(learner$internal_tuned_values, types = "integerish")
+ expect_equal(names(learner$internal_tuned_values), "nrounds")
+
+ learner$validate = NULL
+ expect_error(learner$train(task), "field 'validate'")
+
+ learner$validate = 0.2
+ task$internal_valid_task = NULL
+ learner$param_set$set_values(
+ early_stopping_rounds = NULL
+ )
+ learner$train(task)
+ expect_equal(learner$internal_tuned_values, named_list())
+ expect_named(learner$model$evaluation_log, c("iter", "test_rmse"))
+ expect_list(learner$internal_valid_scores, types = "numeric")
+ expect_equal(names(learner$internal_valid_scores), "rmse")
+
+ learner = lrn("regr.xgboost",
+ nrounds = to_tune(upper = 1000, internal = TRUE),
+ validate = 0.2
+ )
+ s = learner$param_set$search_space()
+ expect_error(learner$param_set$convert_internal_search_space(s), "early stopping")
+ learner$param_set$set_values(early_stopping_rounds = 10)
+ learner$param_set$disable_internal_tuning("nrounds")
+ expect_equal(learner$param_set$values$early_stopping_rounds, NULL)
})