
    ёi(                        S SK Jr  S SKrS SKJr  S SKJrJr  S SKJ	r	  SSK
Jr  SS	KJr  \(       a/  S S
KJr  S SKJr  S SKJr  S SKJr  S SKJr  SSKJr   " S S\5      r/ r " S S\5      rg)    )annotationsN)TYPE_CHECKING)_C_opspir)in_dynamic_or_pir_mode   )	framework   )	Optimizer)Sequence)NotRequired)Tensor)GradientClipBase)WeightDecayRegularizer)_ParameterConfigc                  *    \ rS rSr% S\S'   S\S'   Srg)_AdagradParameterConfig'   zNotRequired[float]epsiloninitial_accumulator_value N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r       X/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/optimizer/adagrad.pyr   r   '   s    ###55r   r   c                     ^  \ rS rSr% SrS\S'   S\S'   Sr      S               SU 4S jjjrS	 rS
 r	S r
SrU =r$ )Adagrad/   ay  
The Adaptive Gradient optimizer (Adagrad for short) use an optimization described
in paper: `Adaptive Subgradient Methods for Online Learning and
Stochastic Optimization <http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf>`_.

The parameter ``param_out`` update rule with gradient ``grad``:

.. math::

    moment\_out &= moment + grad * grad

    param\_out &= param - \frac{learning\_rate * grad}{\sqrt{moment\_out} + \epsilon}


The original paper does not have the ``epsilon`` attribute. It is added here
in our implementation as also proposed `Per-parameter adaptive learning rate
methods <http://cs231n.github.io/neural-networks-3/#ada>`_
for numerical stability to avoid the division by zero error.

Args:
    learning_rate (float|Tensor): The learning rate used to update ``Parameter``.
        It can be a float value or a ``Variable`` with a float type.
    epsilon (float, optional): A small float value for numerical stability.
        The default value is 1e-06.
    parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` to update to minimize ``loss``.
        This parameter is required in dygraph mode. And you can specify different options for
        different parameter groups such as the learning rate, weight decay, etc,
        then the parameters are list of dict. Note that the learning_rate in parameter groups
        represents the scale of base learning_rate.
        The default value is None in static graph mode, at this time all parameters will be updated.
    weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization.
        It canbe a int or float value as coeff of L2 regularization or
        :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
        If a parameter has set regularizer using :ref:`api_paddle_base_param_attr_aramAttr` already,
        the regularization setting here in optimizer will be ignored for this parameter.
        Otherwise, the regularization setting here in optimizer will take effect.
        Default None, meaning there is no regularization.
    grad_clip (GradientClipBase|None, optional): Gradient clipping strategy, it's an instance of
        some derived class of ``GradientClipBase`` . There are three clipping strategies,
        ClipGradByGlobalNorm, ClipGradByNorm and ClipGradByValue. Default None,
        meaning there is no gradient clipping.
    name (str|None, optional): Normally there is no need for user to set this property.
        For more information, please refer to :ref:`api_guide_Name`.
        The default value is None.
    initial_accumulator_value (float, optional): Initial value for moment accumulator.
        The default value is 0.0.

Examples:
    .. code-block:: python

        >>> import paddle

        >>> inp = paddle.rand(shape=[10, 10])
        >>> linear = paddle.nn.Linear(10, 10)
        >>> out = linear(inp)
        >>> loss = paddle.mean(out)
        >>> adagrad = paddle.optimizer.Adagrad(learning_rate=0.1,
        ...         parameters=linear.parameters())
        >>> out.backward()
        >>> adagrad.step()
        >>> adagrad.clear_grad()

        >>> # Note that the learning_rate of linear_2 is 0.01.
        >>> linear_1 = paddle.nn.Linear(10, 10)
        >>> linear_2 = paddle.nn.Linear(10, 10)
        >>> inp = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1)
        >>> out = linear_1(inp)
        >>> out = linear_2(out)
        >>> loss = paddle.mean(out)
        >>> adagrad = paddle.optimizer.Adagrad(
        ...     learning_rate=0.1,
        ...     parameters=[{  # type: ignore
        ...         'params': linear_1.parameters()
        ...     }, {
        ...         'params': linear_2.parameters(),
        ...         'weight_decay': 0.001,
        ...         'learning_rate': 0.1,
        ...     }],
        ...     weight_decay=0.01)
        >>> out.backward()
        >>> adagrad.step()
        >>> adagrad.clear_grad()

strtypefloatr   momentc                   > Uc   eUc   e[         TU ]  UUUUUS9  SU l        X l        SU l        0 U l        Xpl        UUS.U l        g )N)learning_rate
parametersweight_decay	grad_clipnameadagradF)r   r   )super__init__r$   _epsilon_multi_precision_master_weightsr   _default_dict)	selfr(   r   r)   r*   r+   r,   r   	__class__s	           r   r/   Adagrad.__init__   sr     ((("""'!% 	 	
 	 %!)B&)B
r   c                P   [        U[        R                  [        R                  45      (       d  [	        S5      e[        U[
        5      (       a  U R                  U5      nU GH?  nUR                  U R                  ;   a  M   U R                  (       a}  U R                  UR                  5      (       a]  U R                  U5      nU R                  U R                  UU R                  S9  U R                  R!                  UR                  5        M  U R                  UR                  5      (       a'  U R                  (       d  ["        R$                  " S5        U R                  U R                  UU R                  S9  U R                  R!                  UR                  5        GMB     g )Nblock is not instance of Block.)
fill_valuezAccumulating with FP16/BF16 in optimizer can lead to poor accuracy or slow convergence.Consider using multi_precision=True option of the Momentum optimizer.)
isinstancer	   Blockr   	TypeErrordict_update_param_groupr,   _already_create_accumulatorr1   _is_dtype_fp16_or_bf16dtype_create_master_weight_add_accumulator_moment_acc_strr   addwarningswarn)r4   blockr)   pmaster_ps        r   _create_accumulatorsAdagrad._create_accumulators   sJ   %)//399!=>>=>>j$''11*=JAvv999$$)D)DQWW)M)M55a8%%((#== & 
 0044QVV<++AGG44--\ !!$$99 " 
 ,,0083 r   c           	        [        U[        R                  [        R                  45      (       d  [	        S5      e[        U[
        5      (       a  U R                  U5      nU R                  U R                  US   5      nU R                  =(       a    U R                  US   R                  5      nU(       a  U R                  US   R                     OS n[        5       (       aI  [        R                   " US   US   UU R#                  U5      U(       a  UOS U R$                  U5      u      ng US   US   UU R#                  U5      S.nUS   US.nU(       a  XWS'   XXS'   UR'                  U R(                  UUU R$                  US.S	S
9n	U	$ )Nr8   r   r
   )ParamGradMomentLearningRate)ParamOut	MomentOutMasterParamMasterParamOut)r   multi_precisionT)r$   inputsoutputsattrsstop_gradient)r:   r	   r;   r   r<   r=   r>   _get_accumulator_masterrD   r1   r@   rA   r2   r,   r   r   adagrad__create_param_lrr0   	append_opr$   )
r4   rH   param_and_grad
moment_accfind_mastermaster_weight_rW   rX   
adagrad_ops
             r   _append_optimize_opAdagrad._append_optimize_op   s   %)//399!=>>=>>nd++!55nEN11  ."3

 ++ 
0K0K1##1
    !2!7!78 	 "##ooq!q!%%n5!,$GAq!  (*&q)$ $ 5 5n E	F $2!#4:NG(5}%,9()YY#}}'2 # ) 	J r   c                    UR                  SU R                  S   5      U l        UR                  SU R                  S   5      U l        UR                  S5      nU$ )Nr   r   params)getr3   r0   r   )r4   r)   s     r   r>   Adagrad._update_param_group  sX    "y$2D2DY2OP)3':;*
&  ^^H-
r   )r3   r0   r2   r1   r   r$   )gư>NNNNg        )r(   zfloat | Tensorr   r%   r)   z;Sequence[Tensor] | Sequence[_AdagradParameterConfig] | Noner*   z%float | WeightDecayRegularizer | Noner+   zGradientClipBase | Noner,   z
str | Noner   r%   returnNone)r   r   r   r   __doc__r   rD   r/   rK   re   r>   r   __classcell__)r5   s   @r   r!   r!   /   s    Sj I$$O
   >B-1+.
%
 

 H
 <
 +
 
 $)
 

 
>9B9v r   r!   )
__future__r   rF   typingr   paddler   r   paddle.frameworkr   baser	   	optimizerr   collections.abcr   typing_extensionsr   r   paddle.nn.clipr   paddle.regularizerr   r   r   __all__r!   r   r   r   <module>rz      sW    #       (-/9+6"2 6
 \i \r   