
    ёi6                        S SK Jr  S SKrS SKJr  S SKJrJr  S SKJ	r	  SSK
JrJr  SSKJrJr  S	S
KJr  \(       a5  S SKJr  S SKJr  S SKJr  S SKJr  S SKJr  S SKJr  S	SKJr   " S S\5      r/ r " S S\5      r g)    )annotationsN)TYPE_CHECKING)_C_opspir)DataType   )core	framework)in_dynamic_or_pir_modein_pir_mode   )	Optimizer)Sequence)NotRequired)Tensor)GradientClipBase)LRScheduler)WeightDecayRegularizer)_ParameterConfigc                  4    \ rS rSr% S\S'   S\S'   S\S'   Srg)	_RAdamParameterConfig*   zNotRequired[float | Tensor]beta1beta2zNotRequired[float]epsilon N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r       V/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/optimizer/radam.pyr   r   *   s    ****##r#   r   c                     ^  \ rS rSrSrSrSrSrSrSr	        S                 SU 4S jjjr
S	 rS
 rS rS rSrU =r$ )RAdam3   a  
The RAdam optimizer is implemented based on the Adam Optimization
in paper `On the Variance of the Adaptive Learning Rate and Beyond <https://arxiv.org/abs/1908.03265>`_.
RAdam improved the initial stability of training by modifying Adam's momentum term.

.. math::

    \begin{aligned}
        &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
        &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
        &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
        &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
        &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
            2 t \beta^t_2 /\big(1-\beta_2^t \big)                                    \\
        &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
        &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
        &\hspace{12mm} r_t \leftarrow
    \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
        &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} r_t l_t        \\
        &\hspace{6mm}\textbf{else}                                                           \\
        &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}                \\
        &\hspace{0mm} \text{ with: } \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)}, \: \theta_t \text{ (params)} \\
        &\hspace{0mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1
    \end{aligned}

Args:
    learning_rate (float|LRScheduler, optional): The learning rate used to update ``Parameter``.
        It can be a float value or a LRScheduler. The default value is 0.001.
    parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` names to update to minimize ``loss``.
        This parameter is required in dygraph mode. And you can specify different options for
        different parameter groups such as the learning rate, weight decay, etc,
        then the parameters are list of dict. Note that the learning_rate in parameter groups
        represents the scale of base learning_rate.
        The default value is None in static graph mode, at this time all parameters will be updated.
    beta1 (float|Tensor, optional): The exponential decay rate for the 1st moment estimates.
        It should be a float number or a 0-D Tensor with shape [] and data type as float32.
        The default value is 0.9.
    beta2 (float|Tensor, optional): The exponential decay rate for the 2nd moment estimates.
        It should be a float number or a 0-D Tensor with shape [] and data type as float32.
        The default value is 0.999.
    epsilon (float, optional): A small float value for numerical stability.
        The default value is 1e-08.
    weight_decay (int|float|Tensor|WeightDecayRegularizer|None, optional): The weight decay coefficient, it can be int, float or Tensor.
        Default None, meaning there is no regularization.
    grad_clip (GradientClipBase|None, optional): Gradient clipping strategy, it's an instance of
        some derived class of ``GradientClipBase`` . There are three clipping strategies
        ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` ,
        :ref:`api_paddle_nn_ClipGradByValue` ). Default None, meaning there is no gradient clipping.
    name (str|None, optional): Normally there is no need for user to set this property.
        For more information, please refer to :ref:`api_guide_Name`.
        The default value is None.

Note:
    Currently, RAdam doesn't support sparse parameter optimization.

Examples:
    .. code-block:: python

        >>> import paddle

        >>> inp = paddle.rand([10,10], dtype="float32")
        >>> linear = paddle.nn.Linear(10, 10)
        >>> out = linear(inp)
        >>> loss = paddle.mean(out)

        >>> radam = paddle.optimizer.RAdam(
        ...     learning_rate=0.1,
        ...     parameters=linear.parameters()
        ... )
        >>> out.backward()
        >>> radam.step()
        >>> radam.clear_grad()

        >>> # Note that the learning_rate of linear_2 is 0.01.
        >>> linear_1 = paddle.nn.Linear(10, 10)
        >>> linear_2 = paddle.nn.Linear(10, 10)
        >>> inp = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1)
        >>> out = linear_1(inp)
        >>> out = linear_2(out)
        >>> loss = paddle.mean(out)
        >>> opt = paddle.optimizer.RAdam(
        ...     learning_rate=0.1,
        ...     parameters=[{  # type: ignore
        ...         'params': linear_1.parameters()
        ...     }, {
        ...         'params': linear_2.parameters(),
        ...         'weight_decay': 0.001,
        ...         'learning_rate': 0.1,
        ...         'beta1': 0.8
        ...     }],
        ...     weight_decay=0.01,
        ...     beta1=0.9
        ... )
        >>> loss.backward()
        >>> opt.step()
        >>> opt.clear_grad()

	beta1_pow	beta2_powrhomoment1moment2c	                  > [        U[        [        45      (       a  SU::  d  [        SU S35      eSU::  d  [        SU S35      eSUs=::  a  S:  d  O  [        SU S35      eSUs=::  a  S:  d  O  [        S	U S
35      e[        T	U ]  UUUUUS9  SU l        X l        X0l        X@l	        SU l
        0 U l        UUUS.U l        g )Ng        zInvalid learning rate: z, expect learning_rate >= 0.zInvalid epsilon value: z, expect epsilon >= 0.      ?zInvalid beta1: z, expect 0. <= beta1 < 1.0.zInvalid beta2: z, expect 0. <= beta2 < 1.0.)learning_rate
parametersweight_decay	grad_clipnameradamF)r   r   r   )
isinstancefloatint
ValueErrorsuper__init__type_beta1_beta2_epsilon_multi_precision_master_weights_default_dict)
selfr/   r   r   r   r0   r1   r2   r3   	__class__s
            r$   r:   RAdam.__init__   s
    meS\223-;O)-8TU  g~)'2HI  e!c!!%(CD  e!c!!%(CD  	'!% 	 	
 	 %!
r#   c                   UR                   nU R                  U5      (       aC  [        5       (       a  [        R                  O#[
        R                  R                  R                  nU R                  U R                  UUSS9  U R                  U R                  UUSS9  U R                  U R                  UUSS9  U R                  U R                  XS9  U R                  U R                  XS9  g )Nr.   )r3   paramdtype
fill_value)rG   )rG   _is_dtype_fp16_or_bf16r   r   FLOAT32r	   VarDescVarTypeFP32_add_accumulator_beta1_pow_acc_str_beta2_pow_acc_str_rho_acc_str_moment1_acc_str_moment2_acc_str)rB   p	acc_dtypes      r$   _add_moments_powsRAdam._add_moments_pows   s    GG	&&y11$/MM  t||7K7K7P7P  	((	 	 	
 	((	 	 	
 	""	 	 	
 	d33QHd33QHr#   c                    [        U[        R                  [        R                  45      (       d  [	        S5      e[        U[
        5      (       a  UR                  S5      nU GH  nUR                  U R                  ;   a  M   U R                  (       ai  U R                  UR                  5      (       aI  U R                  U5      nU R                  U5        U R                  R                  UR                  5        M  U R                  UR                  5      (       a'  U R                  (       d  [        R                   " S5        U R                  U5        U R                  R                  UR                  5        GM     g )Nblock is not instance of Block.paramszAccumulating with FP16 in optimizer can lead to poor accuracy or slow convergence.Consider using multi_precision=True option of the Lars optimizer.)r5   r
   Blockr   	TypeErrordictgetr3   _already_create_accumulatorr?   rI   rG   _create_master_weightrV   addwarningswarn)rB   blockr0   rT   master_ps        r$   _create_accumulatorsRAdam._create_accumulators   s   %)//399!=>>=>>j$''#1JAvv999$$)D)DQWW)M)M55a8&&x00044QVV<++AGG44--X ""1%,,008% r#   c                f   [        U[        R                  [        R                  45      (       d  [	        S5      e[        U[
        5      (       a  U R                  U5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  U R                  US   5      nU R                  =(       a    U R                  US   R                  5      nU(       a  U R                   US   R"                     OS n	[%        5       (       aV  [&        R(                  " US   US   U R+                  U5      UUUUUU	U R,                  U R.                  U R0                  U5        g US   US   UUUUUU R+                  U5      S.n
US   UUUUUS.nU(       a  XS'   XS'   UR3                  U R4                  U
UU R0                  U R,                  U R.                  S.S	S
9nU$ )NrY   r   r   )rF   gradr(   r)   r*   r+   r,   r/   )	param_outbeta1_pow_outbeta2_pow_outrho_outmoment1_outmoment2_outmaster_parammaster_param_out)r   r   r   T)r;   inputsoutputsattrsstop_gradient)r5   r
   r[   r   r\   r]   _update_param_group_get_accumulator_masterrO   rP   rQ   rR   rS   r?   rI   rG   r@   r3   r   r   radam__create_param_lrr<   r=   r>   	append_opr;   )rB   rd   param_and_gradbeta1_pow_accbeta2_pow_accrho_accmoment1_accmoment2_accfind_mastermaster_weightrr   rs   radam_ops                r$   _append_optimize_opRAdam._append_optimize_op  sX   %)//399!=>>=>>nd++!55nEN44##^A%6
 44##^A%6
 ..~a0
 22!!>!#4
 22!!>!#4
 ++ 
0K0K1##1

    !2!7!78 	 "##MMq!q!%%n5  (*&q)**&&!%!6!6~!F	F ,A.!.!."**G )6~&.;*+YY#}}![[![[
 # ' 
H Or#   c                    UR                  SU R                  S   5      U l        UR                  SU R                  S   5      U l        UR                  SU R                  S   5      U l        UR                  S5      nU$ )Nr   r   r   rZ   )r^   rA   r>   r<   r=   )rB   r0   s     r$   rv   RAdam._update_param_group]  sm    "y$2D2DY2OP nnWd.@.@.IJ nnWd.@.@.IJ^^H-
r#   )r<   r=   rA   r>   r@   r?   r;   )gMbP?g?g+?g:0yE>NNNN)r/   zfloat | LRSchedulerr   float | Tensorr   r   r   r6   r0   z9Sequence[Tensor] | Sequence[_RAdamParameterConfig] | Noner1   z.float | Tensor | WeightDecayRegularizer | Noner2   zGradientClipBase | Noner3   z
str | NonereturnNone)r   r   r   r    __doc__rO   rP   rQ   rR   rS   r:   rV   rf   r   rv   r"   __classcell__)rC   s   @r$   r&   r&   3   s    aF %$L   .3 # % GK-10
*0
 0
 	0

 0
 F0
 E0
 +0
 0
 
0
 0
dI:96Tl r#   r&   )!
__future__r   rb   typingr   paddler   r   paddle.base.libpaddler   baser	   r
   base.frameworkr   r   	optimizerr   collections.abcr   typing_extensionsr   r   paddle.nn.clipr   paddle.optimizer.lrr   paddle.regularizerr   r   r   __all__r&   r   r#   r$   <module>r      s]    #     * " !(-//9+$ 0 $ oI or#   