
    ёi8                        S SK Jr  S SKrS SKJr  S SKJrJr  S SKJ	r	  SSK
JrJr  SSKJrJr  S	S
KJr  \(       a/  S SKJr  S SKJr  S SKJr  S SKJr  S	SKJr  S	SKJr   " S S\5      r/ r " S S\5      rg)    )annotationsN)TYPE_CHECKING)_C_opspir)DataType   )core	framework)in_dynamic_or_pir_modein_pir_mode   )	Optimizer)Sequence)NotRequired)Tensor)GradientClipBase)LRScheduler)_ParameterConfigc                  >    \ rS rSr% S\S'   S\S'   S\S'   S\S'   Srg	)
_NAdamParameterConfig)   zNotRequired[float | Tensor]beta1beta2zNotRequired[float]epsilonmomentum_decay N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r       V/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/optimizer/nadam.pyr   r   )   s    ****##**r#   r   c                     ^  \ rS rSrSrSrSrSrSrSr	         S                   SU 4S jjjr
S	 rS
 rS rS rSrU =r$ )NAdam3   a  
The NAdam optimizer is implemented based on the Adam Optimization
in paper `Incorporating Nesterov Momentum into Adam <https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ>`_.
The main improvement is to combine the advantages of Nesterov momentum and Adam adaptive learning rate.

.. math::

   \begin{aligned}
        &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
        &\hspace{5mm} \mu_t \leftarrow \beta_1 \big(1 - \frac{1}{2}  \rho ^{t \psi} \big)     \\
        &\hspace{5mm} \mu_{t+1} \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96 ^{(t+1)\psi}\big)\\
        &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
        &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
        &\hspace{5mm}\widehat{m_t} \leftarrow \mu_{t+1} m_t/(1-\prod_{i=1}^{t+1}\mu_i) + (1-\mu_t) g_t /(1-\prod_{i=1}^{t} \mu_{i})                         \\
        &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
        &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
            \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
        &\hspace{0mm} \text{ with: } \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)}, \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)} \\
        &\hspace{0mm} \: \lambda \text{ (weight decay)}, \:\psi \text{ (momentum decay)} \\
   \end{aligned}

Args:
    learning_rate (float|LRScheduler, optional): The learning rate used to update ``Parameter``.
        It can be a float value or a LRScheduler. The default value is 0.002.
    parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` names to update to minimize ``loss``.
        This parameter is required in dygraph mode. And you can specify different options for
        different parameter groups such as the learning rate, weight decay, etc,
        then the parameters are list of dict. Note that the learning_rate in parameter groups
        represents the scale of base learning_rate.
        The default value is None in static graph mode, at this time all parameters will be updated.
    beta1 (float|Tensor, optional): The exponential decay rate for the 1st moment estimates.
        It should be a float number or a 0-D Tensor with shape [] and data type as float32.
        The default value is 0.9.
    beta2 (float|Tensor, optional): The exponential decay rate for the 2nd moment estimates.
        It should be a float number or a 0-D Tensor with shape [] and data type as float32.
        The default value is 0.999.
    epsilon (float, optional): A small float value for numerical stability.
        The default value is 1e-08.
    weight_decay (int|float|Tensor|None, optional): The weight decay coefficient, it can be int, float or Tensor.
        Default None, meaning there is no regularization.
    momentum_decay (float, optional): momentum momentum_decay. The default value is 0.004.
    grad_clip (GradientClipBase|None, optional): Gradient clipping strategy, it's an instance of
        some derived class of ``GradientClipBase`` . There are three clipping strategies
        ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` ,
        :ref:`api_paddle_nn_ClipGradByValue` ). Default None, meaning there is no gradient clipping.
    name (str|None, optional): Normally there is no need for user to set this property.
        For more information, please refer to :ref:`api_guide_Name`.
        The default value is None.

Notes:
    Currently, NAdam doesn't support sparse parameter optimization.

Examples:
    .. code-block:: python

        >>> import paddle

        >>> inp = paddle.rand([10,10], dtype="float32")
        >>> linear = paddle.nn.Linear(10, 10)
        >>> out = linear(inp)
        >>> loss = paddle.mean(out)

        >>> nadam = paddle.optimizer.NAdam(
        ...     learning_rate=0.1,
        ...     parameters=linear.parameters()
        ... )
        >>> out.backward()
        >>> nadam.step()
        >>> nadam.clear_grad()

        >>> # Note that the learning_rate of linear_2 is 0.01.
        >>> linear_1 = paddle.nn.Linear(10, 10)
        >>> linear_2 = paddle.nn.Linear(10, 10)
        >>> inp = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1)
        >>> out = linear_1(inp)
        >>> out = linear_2(out)
        >>> loss = paddle.mean(out)
        >>> opt = paddle.optimizer.NAdam(
        ...     learning_rate=0.1,
        ...     parameters=[{  # type: ignore
        ...         'params': linear_1.parameters()
        ...     }, {
        ...         'params': linear_2.parameters(),
        ...         'weight_decay': 0.001,
        ...         'learning_rate': 0.1,
        ...         'beta1': 0.8
        ...     }],
        ...     weight_decay=0.01,
        ...     beta1=0.9
        ... )
        >>> loss.backward()
        >>> opt.step()
        >>> opt.clear_grad()

momentum_decay_pow	beta2_pow
mu_productmoment1moment2c
                  > [        U[        [        45      (       a  SU::  d  [        SU S35      eSU::  d  [        SU S35      eSUs=::  a  S:  d  O  [        SU S35      eSUs=::  a  S:  d  O  [        S	U S
35      eSU::  d  [        SU S35      e[        T
U ]  UUUUU	S9  SU l        X l        X0l        X@l	        XPl
        SU l        0 U l        UUUUS.U l        g )Ng        zInvalid learning rate: z, expect learning_rate >= 0.zInvalid epsilon value: z, expect epsilon >= 0.      ?zInvalid beta1: z, expect 0. <= beta1 < 1.0.zInvalid beta2: z, expect 0. <= beta2 < 1.0.zInvalid momentum_decay value: z, expect momentum_decay >= 0.)learning_rate
parametersweight_decay	grad_clipnamenadamF)r   r   r   r   )
isinstancefloatint
ValueErrorsuper__init__type_beta1_beta2_epsilon_momentum_decay_multi_precision_master_weights_default_dict)selfr/   r   r   r   r   r0   r1   r2   r3   	__class__s             r$   r:   NAdam.__init__   s8    meS\223-;O)-8TU  g~)'2HI  e!c!!%(CD  e!c!!%(CD  n$00@@]^  	'!% 	 	
 	- %!,	
r#   c                   UR                   nU R                  U5      (       aC  [        5       (       a  [        R                  O#[
        R                  R                  R                  nU R                  U R                  UUSS9  U R                  U R                  UUSS9  U R                  U R                  UUSS9  U R                  U R                  XS9  U R                  U R                  XS9  g )Nr.   )r3   paramdtype
fill_value)rH   )rH   _is_dtype_fp16_or_bf16r   r   FLOAT32r	   VarDescVarTypeFP32_add_accumulator_momentum_decay_pow_acc_str_beta2_pow_acc_str_mu_product_acc_str_moment1_acc_str_moment2_acc_str)rC   p	acc_dtypes      r$   _add_moments_powsNAdam._add_moments_pows   s    GG	&&y11$/MM  t||7K7K7P7P  	11	 	 	
 	((	 	 	
 	))	 	 	
 	d33QHd33QHr#   c                    [        U[        R                  [        R                  45      (       d  [	        S5      e[        U[
        5      (       a  UR                  S5      nU GH  nUR                  U R                  ;   a  M   U R                  (       ai  U R                  UR                  5      (       aI  U R                  U5      nU R                  U5        U R                  R                  UR                  5        M  U R                  UR                  5      (       a'  U R                  (       d  [        R                   " S5        U R                  U5        U R                  R                  UR                  5        GM     g )Nblock is not instance of Block.paramszAccumulating with FP16 in optimizer can lead to poor accuracy or slow convergence.Consider using multi_precision=True option of the Lars optimizer.)r5   r
   Blockr   	TypeErrordictgetr3   _already_create_accumulatorr@   rJ   rH   _create_master_weightrW   addwarningswarn)rC   blockr0   rU   master_ps        r$   _create_accumulatorsNAdam._create_accumulators   s   %)//399!=>>=>>j$''#1JAvv999$$)D)DQWW)M)M55a8&&x00044QVV<++AGG44--X ""1%,,008% r#   c                   [        U[        R                  [        R                  45      (       d  [	        S5      e[        U[
        5      (       a  U R                  U5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  =(       a    U R                  US   R                  5      nU(       a  U R                   US   R"                     OS n	[%        5       (       aa  [&        R(                  " US   US   U R+                  U5      UUUUUU	U R,                  U R.                  U R0                  U R2                  U5        g US   US   UUUUUU R+                  U5      S.n
US   UUUUUS.nU(       a  XS'   XS'   UR5                  U R6                  U
UU R0                  U R,                  U R.                  U R2                  S.S	S
9nU$ )NrZ   r   r   )rG   gradr(   r)   r*   r+   r,   r/   )	param_outmomentum_decay_pow_outbeta2_pow_outmu_product_outmoment1_outmoment2_outmaster_parammaster_param_out)r   r   r   r   T)r;   inputsoutputsattrsstop_gradient)r5   r
   r\   r   r]   r^   _update_param_group_get_accumulator_masterrP   rQ   rR   rS   rT   r@   rJ   rH   rA   r3   r   r   nadam__create_param_lrr<   r=   r>   r?   	append_opr;   )rC   re   param_and_gradmomentum_decay_pow_accbeta2_pow_accmu_product_accmoment1_accmoment2_accfind_mastermaster_weightrs   rt   nadam_ops                r$   _append_optimize_opNAdam._append_optimize_op  sk   %)//399!=>>=>>nd++!55nEN!%!=!=,,nQ.?"
 44##^A%6
 55$$nQ&7
 22!!>!#4
 22!!>!#4
 ++ 
0K0K1##1

    !2!7!78 	 "##MMq!q!%%n5&$$   (*&q)&<*,&&!%!6!6~!F	F ,A.*@!."0**G )6~&.;*+YY#}}![[![[&*&:&:	 # ' H Or#   c                H   UR                  SU R                  S   5      U l        UR                  SU R                  S   5      U l        UR                  SU R                  S   5      U l        UR                  SU R                  S   5      U l        UR                  S5      nU$ )Nr   r   r   r   r[   )r_   rB   r>   r<   r=   r?   )rC   r0   s     r$   rw   NAdam._update_param_groupc  s    "y$2D2DY2OP nnWd.@.@.IJ nnWd.@.@.IJ)~~d001AB 
  ^^H-
r#   )r<   r=   rB   r>   rA   r?   r@   r;   )	gMb`?g?g+?g:0yE>gMbp?NNNN)r/   zfloat | LRSchedulerr   float | Tensorr   r   r   r6   r   r6   r0   z9Sequence[Tensor] | Sequence[_NAdamParameterConfig] | Noner1   zfloat | Tensor | Noner2   zGradientClipBase | Noner3   z
str | NonereturnNone)r   r   r   r    __doc__rP   rQ   rR   rS   rT   r:   rW   rg   r   rw   r"   __classcell__)rD   s   @r$   r&   r&   3   s    ^@ #7$&   .3 # % % .2-17
*7
 7
 	7

 7
 7
 F7
 ,7
 +7
 7
 
7
 7
rI:96Vp r#   r&   )
__future__r   rc   typingr   paddler   r   paddle.base.libpaddler   baser	   r
   base.frameworkr   r   	optimizerr   collections.abcr   typing_extensionsr   r   paddle.nn.clipr   lrr   r   r   __all__r&   r   r#   r$   <module>r      sZ    #     * " !(-/++ 0 + xI xr#   