From c167e82f9e87c637785f1ea81a5052942fdadbc6 Mon Sep 17 00:00:00 2001 From: El loza Date: Fri, 24 Jun 2022 18:12:59 +0200 Subject: [PATCH 1/2] Adding mind_delta setting for early stopping in training settings. --- docs/source/user_guide/config/training_settings.rst | 2 ++ recbole/properties/overall.yaml | 1 + recbole/trainer/trainer.py | 11 ++++++++--- recbole/utils/argument_list.py | 2 +- recbole/utils/utils.py | 6 +++--- tests/config/test_config.py | 1 + tests/config/test_overall.py | 6 ++++++ 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/source/user_guide/config/training_settings.rst b/docs/source/user_guide/config/training_settings.rst index 1d4ff583f..3bcc7e347 100644 --- a/docs/source/user_guide/config/training_settings.rst +++ b/docs/source/user_guide/config/training_settings.rst @@ -23,6 +23,8 @@ Training settings are designed to set parameters about model training. evaluated on the valid dataset. Defaults to ``1``. - ``stopping_step (int)`` : The threshold for validation-based early stopping. Defaults to ``10``. +- ``mind_delta (float)`` : any change in the evaluation measure, no matter how fractional, will be considered an improvement. This parameter allows you to set the minimum amount of difference between the best value obtained and the new value that represents an improvement. + Defaults to ``0.001``. - ``clip_grad_norm (dict)`` : The args of `clip_grad_norm_ `_ which will clip gradient norm of model. Defaults to ``None``. - ``loss_decimal_place(int)``: The decimal place of training loss. Defaults to ``4``. diff --git a/recbole/properties/overall.yaml b/recbole/properties/overall.yaml index 984ce1a4a..21411bdce 100644 --- a/recbole/properties/overall.yaml +++ b/recbole/properties/overall.yaml @@ -23,6 +23,7 @@ neg_sampling: uniform: 1 eval_step: 1 stopping_step: 10 +min_delta: 0.001 clip_grad_norm: ~ # clip_grad_norm: {'max_norm': 5, 'norm_type': 2} weight_decay: 0.0 diff --git a/recbole/trainer/trainer.py b/recbole/trainer/trainer.py index 188e20cc1..c7ee09b5a 100644 --- a/recbole/trainer/trainer.py +++ b/recbole/trainer/trainer.py @@ -87,6 +87,7 @@ def __init__(self, config, model): self.clip_grad_norm = config['clip_grad_norm'] self.valid_metric = config['valid_metric'].lower() self.valid_metric_bigger = config['valid_metric_bigger'] + self.min_delta = config['min_delta'] self.test_batch_size = config['eval_batch_size'] self.gpu_available = torch.cuda.is_available() and config['use_gpu'] self.device = config['device'] @@ -355,7 +356,8 @@ def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progre self.best_valid_score, self.cur_step, max_step=self.stopping_step, - bigger=self.valid_metric_bigger + bigger=self.valid_metric_bigger, + min_delta=self.min_delta ) valid_end_time = time() valid_score_output = (set_color("epoch %d evaluating", 'green') + " [" + set_color("time", 'blue') @@ -702,6 +704,7 @@ def __init__(self, config, model): self.stopping_step = config['stopping_step'] self.valid_metric_bigger = config['valid_metric_bigger'] + self.min_delta = config['min_delta'] self.cur_step = 0 self.best_valid_score = -np.inf if self.valid_metric_bigger else np.inf self.best_valid_result = None @@ -801,7 +804,8 @@ def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progre self.best_valid_score, self.cur_step, max_step=self.stopping_step, - bigger=self.valid_metric_bigger + bigger=self.valid_metric_bigger, + min_delta=self.min_delta ) valid_end_time = time() @@ -1108,7 +1112,8 @@ def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progre self.best_valid_score, self.cur_step, max_step=self.stopping_step, - bigger=self.valid_metric_bigger + bigger=self.valid_metric_bigger, + min_delta=self.min_delta ) valid_end_time = time() valid_score_output = (set_color("epoch %d evaluating", 'green') + " [" + set_color("time", 'blue') diff --git a/recbole/utils/argument_list.py b/recbole/utils/argument_list.py index 655f51b08..bf163b60d 100644 --- a/recbole/utils/argument_list.py +++ b/recbole/utils/argument_list.py @@ -24,7 +24,7 @@ 'epochs', 'train_batch_size', 'learner', 'learning_rate', 'neg_sampling', - 'eval_step', 'stopping_step', + 'eval_step', 'stopping_step','min_delta', 'clip_grad_norm', 'weight_decay', 'loss_decimal_place', diff --git a/recbole/utils/utils.py b/recbole/utils/utils.py index 212005363..4c6bb5b12 100644 --- a/recbole/utils/utils.py +++ b/recbole/utils/utils.py @@ -97,7 +97,7 @@ def get_trainer(model_type, model_name): return getattr(importlib.import_module('recbole.trainer'), 'Trainer') -def early_stopping(value, best, cur_step, max_step, bigger=True): +def early_stopping(value, best, cur_step, max_step, bigger=True, min_delta=0.001): r""" validation-based early stopping Args: @@ -121,7 +121,7 @@ def early_stopping(value, best, cur_step, max_step, bigger=True): stop_flag = False update_flag = False if bigger: - if value >= best: + if value >= best and abs(value - best) >= min_delta: cur_step = 0 best = value update_flag = True @@ -130,7 +130,7 @@ def early_stopping(value, best, cur_step, max_step, bigger=True): if cur_step > max_step: stop_flag = True else: - if value <= best: + if value <= best and abs(best - value) >= min_delta: cur_step = 0 best = value update_flag = True diff --git a/tests/config/test_config.py b/tests/config/test_config.py index c698227d2..631153de0 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -45,6 +45,7 @@ def test_default_settings(self): self.assertIsInstance(config['neg_sampling'], dict) self.assertIsInstance(config['eval_step'], int) self.assertIsInstance(config['stopping_step'], int) + self.assertIsInstance(config['min_delta'], float) self.assertIsInstance(config['checkpoint_dir'], str) self.assertIsInstance(config['eval_args'], dict) diff --git a/tests/config/test_overall.py b/tests/config/test_overall.py index 7838ab52a..516fc3b8d 100644 --- a/tests/config/test_overall.py +++ b/tests/config/test_overall.py @@ -85,6 +85,12 @@ def test_stopping_step(self): } self.assertTrue(run_parms({'stopping_step': [0, 1, 2]})) + def test_min_delta(self): + settings = { + 'epochs': 100 + } + self.assertTrue(run_parms({'min_delta': [0.01, 0.001, 0.0001]})) + def test_checkpoint_dir(self): self.assertTrue(run_parms({'checkpoint_dir': ['saved_1/', './saved_2']})) From d2f3952b104d27c4c09123645a66a821039ffca3 Mon Sep 17 00:00:00 2001 From: El loza Date: Fri, 24 Jun 2022 18:54:28 +0200 Subject: [PATCH 2/2] Adding min_delta setting for early stopping in training settings (fix typo) --- docs/source/user_guide/config/training_settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user_guide/config/training_settings.rst b/docs/source/user_guide/config/training_settings.rst index 3bcc7e347..a80bdeb0e 100644 --- a/docs/source/user_guide/config/training_settings.rst +++ b/docs/source/user_guide/config/training_settings.rst @@ -23,7 +23,7 @@ Training settings are designed to set parameters about model training. evaluated on the valid dataset. Defaults to ``1``. - ``stopping_step (int)`` : The threshold for validation-based early stopping. Defaults to ``10``. -- ``mind_delta (float)`` : any change in the evaluation measure, no matter how fractional, will be considered an improvement. This parameter allows you to set the minimum amount of difference between the best value obtained and the new value that represents an improvement. +- ``min_delta (float)`` : any change in the evaluation measure, no matter how fractional, will be considered an improvement. This parameter allows you to set the minimum amount of difference between the best value obtained and the new value that represents an improvement. Defaults to ``0.001``. - ``clip_grad_norm (dict)`` : The args of `clip_grad_norm_ `_ which will clip gradient norm of model. Defaults to ``None``.