feat(learner): use best score in early stopping improvement (#318)

mlr-org · Jan 7, 2025 · 01fe903 · 01fe903
1 parent 1c2a749
commit 01fe903
Show file tree

Hide file tree

Showing 12 changed files with 20 additions and 10 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -6,6 +6,8 @@
 * feat: Added `n_layers` parameter to MLP
 * BREAKING_CHANGE: Early stopping now not uses `epochs - patience` for the internally tuned
   values instead of the trained number of `epochs` as it was before.
+  Also, the improvement is calcualted as the difference between the current and the best score,
+  not the current and the previous score.
 * feat: Added multimodal melanoma example task
 * feat: Added a callback to iteratively unfreeze parameters for finetuning
 * fix: torch learners can now be used with `AutoTuner`

diff --git a/R/CallbackSetEarlyStopping.R b/R/CallbackSetEarlyStopping.R
@@ -6,17 +6,18 @@ CallbackSetEarlyStopping = R6Class("CallbackSetEarlyStopping",
       self$patience = assert_int(patience, lower = 1L)
       self$min_delta = assert_double(min_delta, lower = 0, len = 1L, any.missing = FALSE)
       self$stagnation = 0L
+      self$best_score = NULL
     },
     on_valid_end = function() {
-      if (is.null(self$prev_valid_scores)) {
-        self$prev_valid_scores = self$ctx$last_scores_valid
+      if (is.null(self$ctx$last_scores_valid)) {
         return(NULL)
       }
-      if (is.null(self$ctx$last_scores_valid)) {
+      if (is.null(self$best_score)) {
+        self$best_score = self$ctx$last_scores_valid[[1L]]
         return(NULL)
       }
       multiplier = if (self$ctx$measures_valid[[1L]]$minimize) -1 else 1
-      improvement = multiplier * (self$ctx$last_scores_valid[[1L]] - self$prev_valid_scores[[1L]])
+      improvement = multiplier * (self$ctx$last_scores_valid[[1L]] - self$best_score)
 
       if (is.na(improvement)) {
         lg$warn("Learner %s in epoch %s: Difference between subsequent validation performances is NA",
@@ -32,7 +33,9 @@ CallbackSetEarlyStopping = R6Class("CallbackSetEarlyStopping",
       } else {
         self$stagnation = 0
       }
-      self$prev_valid_scores = self$ctx$last_scores_valid
+      if (improvement > 0) {
+        self$best_score = self$ctx$last_scores_valid[[1L]]
+      }
     }
   )
 )
diff --git a/man/mlr_callback_set.checkpoint.Rd b/man/mlr_callback_set.checkpoint.Rd
diff --git a/man/mlr_callback_set.history.Rd b/man/mlr_callback_set.history.Rd
diff --git a/man/mlr_callback_set.progress.Rd b/man/mlr_callback_set.progress.Rd
diff --git a/man/mlr_learners.mlp.Rd b/man/mlr_learners.mlp.Rd
diff --git a/man/mlr_learners.tab_resnet.Rd b/man/mlr_learners.tab_resnet.Rd
diff --git a/man/mlr_learners.torch_featureless.Rd b/man/mlr_learners.torch_featureless.Rd
diff --git a/man/mlr_learners.torchvision.Rd b/man/mlr_learners.torchvision.Rd
diff --git a/man/mlr_learners_torch.Rd b/man/mlr_learners_torch.Rd
diff --git a/man/mlr_learners_torch_image.Rd b/man/mlr_learners_torch_image.Rd
diff --git a/man/mlr_learners_torch_model.Rd b/man/mlr_learners_torch_model.Rd