From 1f832c15bda63ed019ecc852d173c4de636095f5 Mon Sep 17 00:00:00 2001 From: Quentin Anthony Date: Sat, 23 Sep 2023 14:36:48 -0400 Subject: [PATCH] Fix final value of exponential decay lr --- megatron/learning_rates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/megatron/learning_rates.py b/megatron/learning_rates.py index c70efe7c6..d5d2640c9 100644 --- a/megatron/learning_rates.py +++ b/megatron/learning_rates.py @@ -82,7 +82,8 @@ def get_lr(self): ) elif self.decay_style == "exponential": # exp(-0.693) = 1/2 - lr = self.start_lr * math.exp(-0.693 * num_iters_ / self.end_iter) + end_iter = self.end_iter - self.warmup_iter + lr = self.start_lr * math.exp(-0.693 * num_iters_ / end_iter) else: lr = self.start_lr return max(lr, self.min_lr)