
    RЦi                     :    S r SSKrSSKrSSKJr   " S S\5      rg)a  RAdam Optimizer.
Implementation lifted from: https://github.com/LiyuanLucasLiu/RAdam
Paper: `On the Variance of the Adaptive Learning Rate and Beyond` - https://arxiv.org/abs/1908.03265

NOTE: This impl has been deprecated in favour of torch.optim.RAdam and remains as a reference
    N)	Optimizerc                   v   ^  \ rS rSrSr    SU 4S jjrU 4S jr\R                  " 5       SS j5       r	Sr
U =r$ )	RAdamLegacy   zwPyTorch RAdam optimizer

NOTE: This impl has been deprecated in favour of torch.optim.AdamW and remains as a reference
c                    > [        UUUU[        S5       Vs/ s H  n/ SQPM	     snS9n[        [        U ]  X5        g s  snf )N
   )NNN)lrbetasepsweight_decaybuffer)dictrangesuperr   __init__)	selfparamsr	   r
   r   r   _defaults	__class__s	           O/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/optim/radam.pyr   RAdamLegacy.__init__   sH     %05b	:	1&	:
 	k4)&; ;s   ?c                 ,   > [         [        U ]  U5        g N)r   r   __setstate__)r   stater   s     r   r   RAdamLegacy.__setstate__"   s    k4-e4    c                    S nUb%  [         R                  " 5          U" 5       nS S S 5        U R                   GH  nUS    GH~  nUR                  c  M  UR                  R	                  5       nUR
                  (       a  [        S5      eUR	                  5       nU R                  U   n[        U5      S:X  a8  SUS'   [         R                  " U5      US'   [         R                  " U5      US'   O.US   R                  U5      US'   US   R                  U5      US'   US   US   pUS   u  pU	R                  U5      R                  XUSU-
  S	9  UR                  U
5      R                  USU
-
  S
9  US==   S-  ss'   US   [        US   S-  5         nUS   US   :X  a
  US   US   pOUS   US'   XS   -  nSSU-
  -  S-
  nUSUS   -  U-  SU-
  -  -
  nXS'   US:  aI  US   [        R                   " SU-
  US-
  -  US-
  -  US-
  -  U-  U-  US-
  -  5      -  SXS   -  -
  -  nOUS   SXS   -  -
  -  nXS'   US   S:w  a  UR                  XcS   * US   -  S
9  US:  a5  U	R!                  5       R                  US   5      nUR#                  UUU* S	9  OUR                  X* S
9  UR%                  U5        GM     GM     U$ ! , (       d  f       GN= f)Nr   z'RAdam does not support sparse gradientsr   stepexp_avg
exp_avg_sqr
      )value)alphar   r         r	      r   r   )torchenable_gradparam_groupsgradfloat	is_sparseRuntimeErrorr   len
zeros_liketype_asmul_addcmul_add_intmathsqrtaddcdiv_copy_)r   closurelossgrouppr,   p_fp32r   r!   r"   beta1beta2bufferednum_sma	step_sizebeta2_tnum_sma_maxdenoms                     r   r    RAdamLegacy.step%   s1   ""$y % &&E8_66>vv||~>>&'PQQ

1u:?$%E&M','7'7'?E)$*/*:*:6*BE,'',Y'7'?'?'GE)$*/*=*E*Ef*ME,'&+I&6l8K$W~&//!e)/LU#((QY(?f" ?3uV}r/A+BC=HQK/)1!hqkY"'-HQK#V}4G"#q5y/A"5K)Af,=,G1w;,WWG")QK !|$)$K$))[$q[*-81_>$q[*,34 (( ,7?<3= %= ABESY]DZ@Z	%\	 %*$K1uf7M3M$N	"+QK(A-KK^/D.DuT{.RKS a<&OO-225<@EOOGU9*OEKKzK:s % 'z A %$s   K


K )gMbP?)g?g+?g:0yE>r   r   )__name__
__module____qualname____firstlineno____doc__r   r   r)   no_gradr    __static_attributes____classcell__)r   s   @r   r   r      s<     <"5 ]]_C Cr   r   )rN   r7   r)   torch.optim.optimizerr   r   rI   r   r   <module>rS      s#      +]) ]r   