
    RЦi'                         S r SSKrSSKJrJrJrJr  SSKrSSKr\(       a  SSK	J
r
  O\r
 " S S\R                  R                  5      rg)z}PyTorch MADGRAD optimizer

MADGRAD: https://arxiv.org/abs/2101.11075

Code from: https://github.com/facebookresearch/madgrad
    N)TYPE_CHECKINGAnyCallableOptional)	_params_tc                      ^  \ rS rSrSr     SS\S\S\S\S\S\4U 4S	 jjjr\	S
\4S j5       r
\	S
\4S j5       r\R                  " 5       SS\\/ \4      S
\\   4S jj5       rSrU =r$ )MADGRAD   a  
MADGRAD_: A Momentumized, Adaptive, Dual Averaged Gradient Method for Stochastic
Optimization.

.. _MADGRAD: https://arxiv.org/abs/2101.11075

MADGRAD is a general purpose optimizer that can be used in place of SGD or
Adam may converge faster and generalize better. Currently GPU-only.
Typically, the same learning rate schedule that is used for SGD or Adam may
be used. The overall learning rate is not comparable to either method and
should be determined by a hyper-parameter sweep.

MADGRAD requires less weight decay than other methods, often as little as
zero. Momentum values used for SGD or Adam's beta1 should work here also.

On sparse problems both weight_decay and momentum should be set to 0.

Arguments:
    params (iterable):
        Iterable of parameters to optimize or dicts defining parameter groups.
    lr (float):
        Learning rate (default: 1e-2).
    momentum (float):
        Momentum value in  the range [0,1) (default: 0.9).
    weight_decay (float):
        Weight decay, i.e. a L2 penalty (default: 0).
    eps (float):
        Term added to the denominator outside of the root operation to improve numerical stability. (default: 1e-6).
paramslrmomentumweight_decayepsdecoupled_decayc                    > US:  d  US:  a  [        SU S35      eUS::  a  [        SU S35      eUS:  a  [        SU S35      eUS:  a  [        S	5      e[        UUUUUS
9n[        TU ]  X5        g )Nr      z	Momentum z must be in the range [0,1]zLearning rate z must be positivezWeight decay z must be non-negativezEps must be non-negative)r   r   r   r   r   )
ValueErrordictsuper__init__)	selfr   r   r   r   r   r   defaults	__class__s	           Q/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/optim/madgrad.pyr   MADGRAD.__init__7   s     a<8q=y
2MNOO7~bT1BCDD!}\N:OPQQ7799%+
 	*    returnc                     g)NF r   s    r   supports_memory_efficient_fp16&MADGRAD.supports_memory_efficient_fp16R   s    r   c                     g)NTr   r    s    r   supports_flat_paramsMADGRAD.supports_flat_paramsV   s    r   closurec           
      X   SnUb%  [         R                  " 5          U" 5       nSSS5        U R                   GH  nUS   nUS   U-   nUS   nUS   nSU-
  nUS    GH  n	U	R                  c  M  U	R                  n
US:w  a  U
R                  (       a  [        S	5      eU R                  U	   n[        U5      S
:X  ad  S
US'   [         R                  " U	5      US'   [         R                  " U	5      US'   US
:w  a'  [         R                  " U	5      R                  5       US'   US==   S-  ss'   US   nUS   nU[        R                  " US   5      -  nUS
:w  aP  US   (       a  U	R                  SUS   U-  -
  5        O+U
R                  (       a  [        S5      eU
R                  XS9  U
R                  (       Ga|  U
R                  5       n
U
R!                  5       nU	R#                  U
5      nUR#                  U
5      nUR#                  U
5      nUR!                  5       R%                  S5      R                  U5      nUR!                  5       R'                  UR!                  5       USS9nX-  nUR                  UUS9  UR                  UUS9  UR!                  5       R)                  S5      R                  U5      nUR                  XS9  UR!                  5       R                  XS9  UR'                  UR!                  5       USS9nUR!                  5       R                  USS9  U	R                  USS9  GM  US
:X  a2  UR%                  S5      R                  U5      nU	R'                  UUSS9nOUS   nUR+                  XUS9  UR%                  S5      R                  U5      nUR                  XS9  US
:X  a#  U	R-                  UR'                  UUSS95        GM  UR'                  UUSS9nU	R                  SU-
  5      R                  UUS9  GM     GM     U$ ! , (       d  f       GN= f)zPerforms a single optimization step.

Arguments:
    closure (callable, optional): A closure that reevaluates the model and returns the loss.
Nr   r   r   r   r   r   g        z5momentum != 0 is not compatible with sparse gradientsr   stepgrad_sum_sqsx0r   g      ?z;weight_decay option is not compatible with sparse gradients)alphagUUUUUU?)value)torchenable_gradparam_groupsgrad	is_sparseRuntimeErrorstatelen
zeros_likeclonedetachmathsqrtmul_add_coalesce_valuessparse_maskpowaddcdivpow_addcmul_copy_)r   r&   lossgroupr   r   r   r   ckpr2   r5   r)   r*   lambgrad_valp_maskedgrad_sum_sq_maskeds_maskedrms_masked_valsx0_masked_valsgrad_sqp_kp1_masked_valsrmsr+   zs                             r   r(   MADGRAD.stepZ   s    ""$y % &&E,Cts"B 0LZ(HXB8_66>vvs?t~~&'^__

1u:?$%E&M+0+;+;A+>E-(!&!1!1!!4E#J1}&+kk!n&;&;&=df"#M2#JDIIeFm44  1$./sU4[<%??@>>"./l"mm		!	8>>>==?D#||~H }}T2H)4)@)@)F& }}T2H '9&@&@&B&F&Fu&M&R&RSV&WO%-%5%5%7%?%?@P@P@RTckl%?%mN #kG$$WD$9&++G4+@&8&@&@&B&G&G&N&S&STW&XOFF4F,$$&++H+A )7(>(>x?O?O?QSbjl(>(m%$$&++,=R+HFF82F.1})ooe499#>YYq#QY7"4[  ((4(@%//%055c:C FF4F,  1}

1c
 <=JJq#RJ8 q2v++AR+8] % 'n u %$s   P
P)r   )g{Gz?g?r   gư>F)N)__name__
__module____qualname____firstlineno____doc__r   floatboolr   propertyr!   r$   r/   no_gradr   r   r(   __static_attributes____classcell__)r   s   @r   r	   r	      s    B !"#$)++ + 	+
  + + "+ +6    d   ]]_bHXb%i%89 bXe_ b br   r	   )rZ   r:   typingr   r   r   r   r/   torch.optimtorch.optim.optimizerr   optim	Optimizerr	   r   r   r   <module>rf      s>     9 9  /Ieekk## er   