From 79d316961c761e606ea83d66e133ae3b8ba56941 Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Fri, 20 Dec 2024 16:15:30 +0800 Subject: [PATCH 1/2] Correct resume lr_scheduler step --- deepmd/pd/train/training.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deepmd/pd/train/training.py b/deepmd/pd/train/training.py index 65e35a1c4b..009c06d159 100644 --- a/deepmd/pd/train/training.py +++ b/deepmd/pd/train/training.py @@ -588,15 +588,14 @@ def warm_up_linear(step, warmup_steps): if self.opt_type == "Adam": self.scheduler = paddle.optimizer.lr.LambdaDecay( learning_rate=self.lr_exp.start_lr, - lr_lambda=lambda step: warm_up_linear( - step + self.start_step, self.warmup_steps - ), + lr_lambda=lambda step: warm_up_linear(step, self.warmup_steps), ) self.optimizer = paddle.optimizer.Adam( learning_rate=self.scheduler, parameters=self.wrapper.parameters() ) if optimizer_state_dict is not None and self.restart_training: self.optimizer.set_state_dict(optimizer_state_dict) + self.scheduler.last_epoch -= 1 # need to minus 1 to accurate else: raise ValueError(f"Not supported optimizer type '{self.opt_type}'") From 5fb1509e9d3ed048869ad803f3882f175b53d6fe Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Fri, 20 Dec 2024 19:20:16 +0800 Subject: [PATCH 2/2] refine --- deepmd/pd/train/training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmd/pd/train/training.py b/deepmd/pd/train/training.py index 009c06d159..0f3c7a9732 100644 --- a/deepmd/pd/train/training.py +++ b/deepmd/pd/train/training.py @@ -595,7 +595,7 @@ def warm_up_linear(step, warmup_steps): ) if optimizer_state_dict is not None and self.restart_training: self.optimizer.set_state_dict(optimizer_state_dict) - self.scheduler.last_epoch -= 1 # need to minus 1 to accurate + self.scheduler.last_epoch -= 1 else: raise ValueError(f"Not supported optimizer type '{self.opt_type}'")