From 59a9a31c0c6b024cd8a456b6d60020e8ef22415a Mon Sep 17 00:00:00 2001 From: lucidrains Date: Wed, 31 Jul 2024 14:45:39 +0000 Subject: [PATCH] comment --- adam_atan2_pytorch/foreach.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adam_atan2_pytorch/foreach.py b/adam_atan2_pytorch/foreach.py index fc8aab7..ecceae1 100644 --- a/adam_atan2_pytorch/foreach.py +++ b/adam_atan2_pytorch/foreach.py @@ -128,7 +128,7 @@ def step( # decay running averages torch._foreach_lerp_(exp_avgs, grads, 1. - beta1) - torch._foreach_lerp_(exp_avg_sqs, grad_squared, 1. - beta2) # grads is grad squared now + torch._foreach_lerp_(exp_avg_sqs, grad_squared, 1. - beta2) # clone for update