
    RЦi                     6    S SK r S SKrS SKJr   " S S\5      rg)    N)	Optimizerc                   l   ^  \ rS rSrSr     SU 4S jjr\R                  " 5       SS j5       rSr	U =r
$ )NAdamLegacy   a  Implements Nadam algorithm (a variant of Adam based on Nesterov momentum).

NOTE: This impl has been deprecated in favour of torch.optim.NAdam and remains as a reference

It has been proposed in `Incorporating Nesterov Momentum into Adam`__.

Arguments:
    params (iterable): iterable of parameters to optimize or dicts defining
        parameter groups
    lr (float, optional): learning rate (default: 2e-3)
    betas (Tuple[float, float], optional): coefficients used for computing
        running averages of gradient and its square
    eps (float, optional): term added to the denominator to improve
        numerical stability (default: 1e-8)
    weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
    schedule_decay (float, optional): momentum schedule decay (default: 4e-3)

__ http://cs229.stanford.edu/proj2015/054_report.pdf
__ http://www.cs.toronto.edu/~fritz/absps/momentum.pdf

    Originally taken from: https://github.com/pytorch/pytorch/pull/1408
    NOTE: Has potential issues but does work well on some problems.
c                    > SU::  d  [        SR                  U5      5      e[        UUUUUS9n[        [        U ]  X5        g )Ng        zInvalid learning rate: {})lrbetasepsweight_decayschedule_decay)
ValueErrorformatdictsuperr   __init__)	selfparamsr   r	   r
   r   r   defaults	__class__s	           O/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/optim/nadam.pyr   NAdamLegacy.__init__    sM     by8??CDD%)
 	k4)&;    c           	      2   SnUb%  [         R                  " 5          U" 5       nSSS5        U R                   GH  nUS    GH  nUR                  c  M  UR                  nU R                  U   n[        U5      S:X  a<  SUS'   SUS'   [         R                  " U5      US'   [         R                  " U5      US'   US   nUS	   nUS   US   pUS
   u  pUS   nUS==   S-  ss'   US   nSX-  -
  nUS   S:w  a  UR                  XCS   S9nUSSSX-  -  -  -
  -  nUSSSUS-   U-  -  -  -
  -  nUU-  nUU-  U-  nUUS'   U	R                  U5      R                  USU-
  S9  U
R                  U5      R                  XUSU-
  S9  U
R                  5       [        R                  " U5      -  R                  U5      nUR                  UUUS   * SU-
  -  SU-
  -  S9  UR                  U	UUS   * U-  SU-
  -  S9  GM     GM     U$ ! , (       d  f       GN= f)zPerforms a single optimization step.

Arguments:
    closure (callable, optional): A closure that reevaluates the model
        and returns the loss.
Nr   r   stepg      ?
m_scheduleexp_avg
exp_avg_sqr   r	   r
      r   )alphag      ?gQ?)valuer   )torchenable_gradparam_groupsgradstatelen
zeros_likeaddmul_add_addcmul_sqrtmathaddcdiv_)r   closurelossgrouppr$   r%   r   r   r   r   beta1beta2r
   tbias_correction2momentum_cache_tmomentum_cache_t_1m_schedule_newm_schedule_nextdenoms                        r   r   NAdamLegacy.step4   sn    ""$y % &&E8_66>vv

1 u:?$%E&M*,E,'','7'7':E)$*/*:*:1*=E,' #<0
!&'7!8&+I&6l8K$W~Elf"&M#$uz> (A-88A>-B8CD#(BAS8T1U,U#V %*b3$AE^C[:\3].]%^"!+.>!>",/?"?BT"T&4l# U#((R%Z(@&//"u*/M#*TYY7G-HHNNsS

4uT{lbCS>S.TXZ]kXk.l
m

7E%+@R1RVX[jVj1k
lM % 'R Y %$s   H
H )gMb`?)g?g+?g:0yE>r   gMbp?)N)__name__
__module____qualname____firstlineno____doc__r   r!   no_gradr   __static_attributes____classcell__)r   s   @r   r   r      s8    6 <( ]]_5 5r   r   )r-   r!   torch.optim.optimizerr   r   r=   r   r   <module>rG      s      +c) cr   