
    ёi]7                        S SK Jr  S SKrS SKJr  S SKrS SKJrJr  S SKJ	r	  SSK
Jr  SSKJr  SS	KJrJr  S
SKJr  \(       a$  S SKJr  S SKJr  S SKJr  S SKJr  S
SKJr  S
SKJr  / r " S S\5      rg)    )annotationsN)TYPE_CHECKING)_C_opspir)	to_tensor   )	framework)no_grad)in_dygraph_modein_pir_mode   )	Optimizer)Sequence)Tensor)GradientClipBase)WeightDecayRegularizer)LRScheduler)_ParameterConfigc                     ^  \ rS rSr% SrS\S'   SrSrSr       S               SU 4S jjjr	S	 r
S
 r\S 5       rS rSrU =r$ )ASGD)   a  
Optimizer of the ASGD algorithm.Please refer to this for details:
`Minimizing Finite Sums with the Stochastic Average Gradient <https://hal.science/hal-00860051v2>`_.

.. math::

   \begin{aligned}
        &\hspace{0mm} d=0,\ y_i=0\ \textbf{for}\ i=1,2,...,n                            \\
        &\hspace{0mm} \textbf{for}\  \: m=0,1,...\ \textbf{do} \:                       \\
        &\hspace{5mm} i=m\ \%\ n                                                        \\
        &\hspace{5mm} d=d-y_i+f_i{}'(x)                                                 \\
        &\hspace{5mm} y_i=f_i{}'(x)                                                     \\
        &\hspace{5mm} x=x-learning\_rate(\frac{d}{\mathrm{min}(m+1,\ n)}+\lambda x)     \\
        &\hspace{0mm} \textbf{end for}                                                  \\
   \end{aligned}

Parameters:
    learning_rate (float|Tensor|LRScheduler, optional): The learning rate used to update ``Parameter``.
        It can be a float value, a ``Tensor`` with a float type or a LRScheduler. The default value is 0.001.
    batch_num (int, optional): The number of batches needed to complete one epoch.
        Assuming the total number of samples is ``all``,
        it is recommended to set ``batch_num`` to ``all`` / ``batch_size``.
        In situations where the graphics memory is tight,
        it is possible to reduce the batch_num appropriately.
        The default value is 1.
    parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` to update to minimize ``loss``.
        This parameter is required in dygraph mode.
        The default value is None in static graph mode, at this time all parameters will be updated.
    weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization.
        It can be a int or float value as coeff of L2 regularization or :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
        If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already,
        the regularization setting here in optimizer will be ignored for this parameter.
        Otherwise, the regularization setting here in optimizer will take effect.
        Default None, meaning there is no regularization.
    grad_clip (GradientClipBase|None, optional): Gradient clipping strategy, it's an instance of some derived class of ``GradientClipBase`` .
        There are three clipping strategies ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` ).
        Default None, meaning there is no gradient clipping.
    multi_precision (bool, optional): In mixed precision training scenarios based on GPU,
        this parameter is mainly used to ensure the numerical stability of gradient updates.
        When it is set to True, the optimizer will save a backup of FP32 type parameters with an equal value for FP16 type parameters.
        When updating gradients, first increase the gradient type to FP32, and then assign it to the FP32 type parameter backup.
        Finally, the updated FP32 type value will be converted to FP16 type first,
        and then assigned to the actual FP16 type parameters participating in the calculation.
        The default value is False.
    name (str|None, optional): The default value is None. Normally there is no need for user to set this property.
        For more information, please refer to :ref:`api_guide_Name` .

Examples:
    .. code-block:: python

        >>> import paddle

        >>> inp = paddle.uniform(min=-0.1, max=0.1, shape=[10, 10], dtype='float32')
        >>> linear = paddle.nn.Linear(10, 10)
        >>> inp = paddle.to_tensor(inp)
        >>> out = linear(inp)
        >>> loss = paddle.mean(out)
        >>> asgd = paddle.optimizer.ASGD(
        ...     learning_rate=0.001,
        ...     batch_num=10,
        ...     parameters=linear.parameters(),
        ...     weight_decay=0.01
        ... )
        >>> out.backward()
        >>> asgd.step()
        >>> asgd.clear_grad()
strtypedymc                   > Uc  [        S5      eUc  [        S5      eSU:  d  [        S5      e[        TU ]	  UUUUUS9  SU l        X`l        0 U l        X l        S U l        g )Nz learning_rate should not be nonezbatch_num should not be noner   z"batch_num should be greater than 0)learning_rate
parametersweight_decay	grad_clipnameasgd)
ValueErrorsuper__init__r   _multi_precision_master_weights_n	_n_tensor)	selfr   	batch_numr   r    r!   multi_precisionr"   	__class__s	           U/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/optimizer/asgd.pyr&   ASGD.__init__s   s      ?@@;<<9}ABB'!% 	 	
 	 /!    c                   [        U[        R                  [        R                  45      (       d  [	        S5      e[        U[
        5      (       a  U R                  U5      nU GHc  nUR                  U R                  ;   a  M   UnU R                  (       a3  U R                  UR                  5      (       a  U R                  U5      nUnU R                  UR                  5      (       a'  U R                  (       d  [        R                  " S5        U R                  U R                   UUR                  S5        U R                  U R"                  UUR                  SU R$                  /['        UR(                  5      Q5        U R                  U R*                  USSS/5        U R                  R-                  UR                  5        GMf     g )Nzblock is not instance of Block.zAccumulating with FP16/BF16 in optimizer can lead to poor accuracy or slow convergence.Consider using multi_precision=True option of the Adam optimizer.r   int64r   )
isinstancer	   Blockr   	TypeErrordict_update_param_groupr"   _already_create_accumulatorr'   _is_dtype_fp16_or_bf16dtype_create_master_weightwarningswarn_add_accumulator
_d_acc_str
_y_acc_strr)   listshape
_m_acc_stradd)r+   blockr   pp_newmaster_ps         r/   _create_accumulatorsASGD._create_accumulators   sj   %)//399!=>>=>>j$''11*=JAvv999E$$)D)DQWW)M)M55a8 ++AGG44--X
 !!	 !!)4=) !! ,,008Q r1   c                   U R                   b  U R                   S-   U-   nU R                  =(       a    U R                  UR                  5      nU(       a  U R                  UR
                     OUnUR
                  nX R                  ;  d  XR                  U   ;  a  [        SU SU 35      e[        5       (       aT  Uc'  [        R                  " U5      U R                  U   U'   g [        R                  " U5      U R                  U   U   U'   g [        U[        R                  5      (       d   eSU0n	SU R                  U   U   0n
UR                  SU	U
S9  g )N_zAccumulator z does not exist for parameter XOutassignr   inputsoutputs)_namer'   r:   r;   r(   r"   _accumulators	Exceptionr   paddlerP   r4   r	   r5   	append_op)r+   rF   r"   paramassign_valueindexfind_mastertarget_paramtarget_nameassign_inputsassign_outputss              r/   _assign_accumulator_masterASGD._assign_accumulator_master   se    ::!::#d*D++ 
0K0KKK1
 1<D  , 	 #''***"4"4T"::tf$B;-P  ==}8> 9""4(5 @F}} @""4(5e< eY__5555 \M
 t))$/<N OO$&  r1   c                B   [        U[        5      (       a  U R                  U5      nU R                  c  [	        U R
                  /5      U l        U R                  U R                  US   5      nU R                  U R                  US   5      nU R                  U R                  US   5      n[        R                  " X@R                  5      R                  5       n[        R                  " XV   5      nU R                  =(       a    U R                  US   R                   5      nU(       a  U R"                  US   R$                     OS n	U R'                  U5      n
[)        5       (       ah  UR+                  [	        S/UR                   S95        [,        R.                  " US   US   U
UXV   [        R0                  " X@R                  5      U	U5        g [3        5       (       a  [        R                  " [        R4                  " U[	        S/UR                   S95      5      nU R7                  XR                  US   US 5        US   n[,        R.                  " US   US   U
UU[        R0                  " X@R                  5      U	U5        U R7                  XR                  US   Xv5        g [        U[8        R:                  5      (       d   eU[	        S/UR                   S9S.nSU0nUR=                  SUUS9  US   nUS   US   U
UU[        R0                  " X@R                  5      S.nUS   UUS	.nS
U0nU(       a  XS'   XS'   UR=                  U R>                  UUUSS9n[        R@                  RC                  XVU5      nU R7                  XR                  US   US 5        U$ )Nr   r   )r;   )rN   YrO   elementwise_addrQ   )rY   gradr   r   r   n)	param_outd_outy_outr-   master_parammaster_param_outT)r   rR   rS   attrsstop_gradient)"r4   r7   r8   r*   r   r)   _get_accumulator_masterr@   rD   rA   rW   moditemrP   r'   r:   r;   r(   r"   _create_param_lrr   add_r   asgd_fminr   rE   ra   r	   r5   rX   r   staticsetitem)r+   rF   param_and_gradr   r   ysr[   r   r\   master_weightlr
add_inputsadd_outputsasgd_inputsasgd_outputs
asgd_attrsasgd_ops                    r/   _append_optimize_opASGD._append_optimize_op   ss   nd++!55nEN>>!&	DN ((.:KL((.:KL))$//>!;LM

1nn-224MM")$++ 
0K0K1##1

    !2!7!78 	 "">2FF9aS01LLq!q!	A~~.	 ]]fjjIqc,IJKA++q(91d !ALLq!q!A~~.	 ++q(91 eY__5555 s!''2J qK OO&!#   !A (*&q)!#[[NN3K ,A.L ,[9J.;N+3@/0ooYY"$ " & G &&r!4B++q(92t Nr1   c                (    UR                  S5      nU$ )Nparams)get)r+   r   s     r/   r8   ASGD._update_param_groupy  s    ^^H-
r1   )r(   r'   r)   r*   r   )gMbP?r   NNNFN)r   zfloat | Tensor | LRSchedulerr,   intr   z4Sequence[Tensor] | Sequence[_ParameterConfig] | Noner    z%float | WeightDecayRegularizer | Noner!   zGradientClipBase | Noner-   boolr"   z
str | NonereturnNone)__name__
__module____qualname____firstlineno____doc____annotations__r@   rA   rD   r&   rJ   ra   r
   r   r8   __static_attributes____classcell__)r.   s   @r/   r   r   )   s    BH IJJJ 7<KO>B-1 %3  I	
 < +   
 :.9`,\ H HT r1   r   )
__future__r   r=   typingr   rW   r   r   paddle.tensor.creationr   baser	   base.dygraphr
   base.frameworkr   r   	optimizerr   collections.abcr   r   paddle.nn.clipr   paddle.regularizerr   r{   r   r   __all__r    r1   r/   <module>r      sM    #      ,  " 9  (/9+
R9 Rr1   