
    ёizK                   n   S SK Jr  S SKrS SKrS SKJr  S SKrS SKJ	r	J
r
Jr  S SKJr  S SKJr  SSKJr  SS	KJr  SS
KJrJrJrJrJrJr  SSKJr  SSKJrJrJ r J!r!  SSK"J#r#J$r$  SSK%J&r&  \(       a(  S SK'J(r(  S SKJ)r)  S SKJ*r*  S SK+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3J4r4  / r5 " S S\&5      r6 " S S\65      r7 " S S\65      r8 " S S\65      r9 " S S\&5      r: " S S \&5      r; " S! S"\&5      r< " S# S$\&5      r= " S% S&\<5      r> " S' S(\<5      r? " S) S*\<5      r@ " S+ S,\<5      rA " S- S.\&5      rB " S/ S0\&5      rCg)1    )annotationsN)TYPE_CHECKING)_C_opsin_dynamic_mode	pir_utils)get_all_custom_device_type)param_one_alias   )dygraph_utils)check_variable_and_dtype)	ParamAttr_global_flagsget_default_dtypein_dynamic_or_pir_modein_pir_modeno_grad   )
functional)
batch_norm
group_norminstance_norm
layer_norm)ConstantNormal   )Layer)Sequence)Literal)Tensor)	DataLayout0DDataLayout1DDataLayout2DDataLayout3DDataLayoutND	DTypeLikeParamAttrLike	PlaceLike	ShapeLikec                     ^  \ rS rSr% SrS\S'   S\S'         S               SU 4S jjjrS rSS jrSS	 jr	S
r
U =r$ )_InstanceNormBase?   z
This class is based class for InstanceNorm1D, 2d, 3d.

See InstanceNorm1D, InstanceNorm2D or InstanceNorm3D for more details.
Tensor | Nonescalebiasc                t  > [         TU ]  5         USL d  USL a  XE:X  d   S5       eX0l        X l        X@l        XPl        Xl        X`l        USLa\  USLaW  U R                  U R                  U/[        S5      SS9U l
        U R                  U R
                  U/[        S5      SS9U l        g S U l
        S U l        g )NFzOweight_attr and bias_attr must be set to False at the same time in InstanceNorm      ?)attrshapedefault_initializeris_bias        T)super__init__	_momentum_epsilon_weight_attr
_bias_attr_num_features_data_formatcreate_parameterr   r-   r.   	selfnum_featuresepsilonmomentumweight_attr	bias_attrdata_formatname	__class__s	           T/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/nn/layer/norm.pyr7   _InstanceNormBase.__init__I   s     	%9#5+ a+ "'#)'e#	(>..&&#n$,SM	 / DJ --__#n$,SM	 . DI DJDI    c                    [        S5      e)NzInstanceNorm Base errorNotImplementedErrorr@   inputs     rI   _check_input_dim"_InstanceNormBase._check_input_dimq   s    !";<<rK   c           	         U R                  U5        [        UU R                  U R                  U R                  U R
                  U R                  S9$ )N)weightr.   rC   epsrF   )rQ   r   r-   r.   r8   r9   r=   rO   s     rI   forward_InstanceNormBase.forwardt   sE    e$::^^))
 	
rK   c                :    SU R                    SU R                   3$ )Nnum_features=
, epsilon=)r<   r9   r@   s    rI   
extra_repr_InstanceNormBase.extra_repr   s     t112*T]]OLLrK   )r;   r=   r9   r8   r<   r:   r.   r-   h㈵>?NNNCHWN)rA   intrB   floatrC   rc   rD   ParamAttrLike | NonerE   rd   rF   zLiteral['NCHW']rG   
str | NonereturnNonerP   r   rf   r   rf   str)__name__
__module____qualname____firstlineno____doc____annotations__r7   rQ   rV   r\   __static_attributes____classcell__rH   s   @rI   r*   r*   ?   s     

 ,0*.'-&& & 	&
 *& (& %& & 
& &P=

M MrK   r*   c                  h   ^  \ rS rSrSr      S               SU 4S jjjrSS jrSrU =r$ )	InstanceNorm1D   av  
Create a callable object of `InstanceNorm1D`. Applies Instance Normalization over a 3D input (a mini-batch of 1D inputs with additional channel dimension) as described in the paper Instance Normalization: The Missing Ingredient for Fast Stylization .

DataLayout: NCL `[batch, in_channels, length]`

:math:`input` is the input features over a mini-batch.

..  math::

    \mu_{\beta} &\gets \frac{1}{HW} \sum_{i=1}^{HW} x_i \qquad &//\
    \ mean\ of\ one\  feature\ map\ in\ mini-batch \\
    \sigma_{\beta}^{2} &\gets \frac{1}{HW} \sum_{i=1}^{HW}(x_i - \
    \mu_{\beta})^2 \qquad &//\ variance\ of\ one\ feature\ map\ in\ mini-batch \\
    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
    \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift

Where `H` means height of feature map, `W` means width of feature map.

Parameters:
    num_features(int): Indicate the number of channels of the input ``Tensor``.
    epsilon(float, optional): A value added to the denominator for
        numerical stability. Default is 1e-5.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for Parameter `scale` of instance_norm.
        If it is set to None or one attribute of ParamAttr, instance_norm
        will create ParamAttr as weight_attr, the name of scale can be set in ParamAttr.
        If the Initializer of the weight_attr is not set, the parameter is initialized
        one. If it is set to False, will not create weight_attr. Default: None. For more information, please refer to :ref:`api_paddle_ParamAttr` .
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the bias of instance_norm.
        If it is set to None or one attribute of ParamAttr, instance_norm
        will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr.
        If the Initializer of the bias_attr is not set, the bias is initialized zero.
        If it is set to False, will not create bias_attr. Default: None. For more information, please refer to :ref:`api_paddle_ParamAttr` .
    data_format(str, optional): Specify the input data format, may be "NC", "NCL". Default "NCL".
    name(str|None, optional): Name for the InstanceNorm, default is None. For more information, please refer to :ref:`api_guide_Name` .


Shape:
    - x: 2-D or 3-D tensor with shape: (batch, num_features) or (batch, num_features, length).
    - output: 3-D tensor with same shape as input x.

Returns:
    None.


Examples:

    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.rand((2, 2, 3))
        >>> instance_norm = paddle.nn.InstanceNorm1D(2)
        >>> instance_norm_out = instance_norm(x)

        >>> print(instance_norm_out)
        Tensor(shape=[2, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[ 1.32132232, -0.22444785, -1.09687424],
          [ 1.29506636, -0.15688568, -1.13818073]],
         [[-0.27764025,  1.33961368, -1.06197333],
          [ 0.44484580, -1.38489723,  0.94005162]]])
c           	     0   > [         TU ]  UUUUUUU5        g Nr6   r7   r?   s	           rI   r7   InstanceNorm1D.__init__   '     		
rK   c                    [        UR                  5      S:w  a<  [        UR                  5      S:w  a"  [        S[        UR                  5       S35      eg g Nr   r
   zexpected 2D or 3D input (got D input)lenr2   
ValueErrorrO   s     rI   rQ   InstanceNorm1D._check_input_dim   N    u{{q S%5%:/EKK0@/AJ  &; rK    )r_   r`   NNNCLN)rA   rb   rB   rc   rC   rc   rD   bool | ParamAttr | NonerE   r   rF   zLiteral['NC', 'NCL']rG   re   rf   rg   rP   r   rf   rg   	rk   rl   rm   rn   ro   r7   rQ   rq   rr   rs   s   @rI   ru   ru      s    >F /3-1,1

 
 	

 -
 +
 *
 
 

 
( rK   ru   c                  h   ^  \ rS rSrSr      S               SU 4S jjjrSS jrSrU =r$ )	InstanceNorm2D   aG  
Create a callable object of `InstanceNorm2D`. Applies Instance Normalization over a 4D input (a mini-batch of 2D inputs with additional channel dimension) as described in the paper Instance Normalization: The Missing Ingredient for Fast Stylization .

DataLayout: NCHW `[batch, in_channels, in_height, in_width]`


:math:`input` is the input features over a mini-batch.

..  math::

    \mu_{\beta} &\gets \frac{1}{HW} \sum_{i=1}^{HW} x_i \qquad &//\
    \ mean\ of\ one\  feature\ map\ in\ mini-batch \\
    \sigma_{\beta}^{2} &\gets \frac{1}{HW} \sum_{i=1}^{HW}(x_i - \
    \mu_{\beta})^2 \qquad &//\ variance\ of\ one\ feature\ map\ in\ mini-batch \\
    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
    \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift

Where `H` means height of feature map, `W` means width of feature map.

Parameters:
    num_features(int): Indicate the number of channels of the input ``Tensor``.
    epsilon(float, optional): A value added to the denominator for
        numerical stability. Default is 1e-5.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for Parameter `scale`
        of instance_norm. If it is set to None or one attribute of ParamAttr, instance_norm
        will create ParamAttr as weight_attr, the name of scale can be set in ParamAttr.
        If the Initializer of the weight_attr is not set, the parameter is initialized
        one. If it is set to False, will not create weight_attr. Default: None. For more information, please refer to :ref:`api_paddle_ParamAttr` .
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the bias of instance_norm.
        If it is set to None or one attribute of ParamAttr, instance_norm
        will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr.
        If the Initializer of the bias_attr is not set, the bias is initialized zero.
        If it is set to False, will not create bias_attr. Default: None. For more information, please refer to :ref:`api_paddle_ParamAttr` .
    data_format(str, optional): Specify the input data format, could be "NCHW". Default: NCHW.
    name(str|None, optional): Name for the InstanceNorm, default is None. For more information, please refer to :ref:`api_guide_Name` .

Shape:
    - x: 4-D tensor with shape: (batch, num_features, height, weight).
    - output: 4-D tensor with same shape as input x.

Returns:
    None.


Examples:

    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.rand((2, 2, 2, 3))
        >>> instance_norm = paddle.nn.InstanceNorm2D(2)
        >>> instance_norm_out = instance_norm(x)

        >>> print(instance_norm_out)
        Tensor(shape=[2, 2, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[[ 1.26652932, -0.60229748, -1.65705574],
           [ 1.06272733,  0.24229208, -0.31219524]],
          [[-0.85414171,  0.31684181, -1.42204332],
           [ 1.00412714, -0.43966094,  1.39487720]]],
         [[[ 0.83324969,  1.25046813, -0.79470295],
           [-1.38446140,  0.81851846, -0.72307163]],
          [[-0.33560610,  0.95346332,  0.45585334],
           [-0.53483474,  1.20336461, -1.74224067]]]])
c           	     0   > [         TU ]  UUUUUUU5        g rx   ry   r?   s	           rI   r7   InstanceNorm2D.__init__%  r{   rK   c                z    [        UR                  5      S:w  a"  [        S[        UR                  5       S35      eg N   zexpected 4D input (got r~   r   rO   s     rI   rQ   InstanceNorm2D._check_input_dim9  ;    u{{q )#ekk*:);8D  !rK   r   r^   )rA   rb   rB   rc   rC   rc   rD   r   rE   r   rF   zLiteral['NCWH']rG   re   rf   rg   r   r   rs   s   @rI   r   r      s    BN /3-1'-

 
 	

 -
 +
 %
 
 

 
( rK   r   c                  h   ^  \ rS rSrSr      S               SU 4S jjjrSS jrSrU =r$ )	InstanceNorm3Di@  a  
Create a callable object of `InstanceNorm3D`. Applies Instance Normalization over a 5D input (a mini-batch of 3D inputs with additional channel dimension) as described in the paper Instance Normalization: The Missing Ingredient for Fast Stylization .

DataLayout: NCDHW `[batch, in_channels, D, in_height, in_width]`


:math:`input` is the input features over a mini-batch.

..  math::

    \mu_{\beta} &\gets \frac{1}{HW} \sum_{i=1}^{HW} x_i \qquad &//\
    \ mean\ of\ one\  feature\ map\ in\ mini-batch \\
    \sigma_{\beta}^{2} &\gets \frac{1}{HW} \sum_{i=1}^{HW}(x_i - \
    \mu_{\beta})^2 \qquad &//\ variance\ of\ one\ feature\ map\ in\ mini-batch \\
    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
    \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift

Where `H` means height of feature map, `W` means width of feature map.

Parameters:
    num_features(int): Indicate the number of channels of the input ``Tensor``.
    epsilon(float, optional): A value added to the denominator for
        numerical stability. Default is 1e-5.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
        of instance_norm. If it is set to None or one attribute of ParamAttr, instance_norm
        will create ParamAttr as weight_attr, the name of scale can be set in ParamAttr.
        If the Initializer of the weight_attr is not set, the parameter is initialized
        one. If it is set to False, will not create weight_attr. Default: None. For more information, please refer to :ref:`api_paddle_ParamAttr` .
    bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of instance_norm.
        If it is set to None or one attribute of ParamAttr, instance_norm
        will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr.
        If the Initializer of the bias_attr is not set, the bias is initialized zero.
        If it is set to False, will not create bias_attr. Default: None. For more information, please refer to :ref:`api_paddle_ParamAttr` .
    data_format(str, optional): Specify the input data format, could be "NCDHW". Default: NCDHW.
    name(str, optional): Name for the InstanceNorm, default is None. For more information, please refer to :ref:`api_guide_Name` .

Shape:
    - x: 5-D tensor with shape: (batch, num_features, dims, height, weight).
    - output: 5-D tensor with same shape as input x.

Returns:
    None.


Examples:

    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.rand((2, 2, 2, 2, 3))
        >>> instance_norm = paddle.nn.InstanceNorm3D(2)
        >>> instance_norm_out = instance_norm(x)

        >>> print(instance_norm_out)
        Tensor(shape=[2, 2, 2, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[[[ 0.60520107, -0.67670596, -1.40020907],
            [ 0.46540472, -0.09736639, -0.47771260]],
           [[-0.74365318,  0.63718963, -1.41333199],
            [ 1.44764769, -0.25489071,  1.90842640]]],
          [[[ 1.09773374,  1.49568439, -0.45503727],
            [-1.01755965,  1.08368278, -0.38671401]],
           [[-0.62252384,  0.60490805,  0.13109155],
            [-0.81222630,  0.84286022, -1.96189928]]]],
         [[[[ 0.28014541,  0.91674680,  1.71797717],
            [-0.52062720, -0.74274176, -0.86439967]],
           [[ 0.25707796, -1.23866379,  1.64422870],
            [-1.48577297, -0.13187379,  0.16790220]]],
          [[[-1.49266160,  1.57909954,  0.46455818],
            [-0.14981404,  1.46959865,  0.24957968]],
           [[ 0.25134835, -0.03276967, -0.30318922],
            [ 0.76263177, -1.11345232, -1.68492818]]]]])
c           	     0   > [         TU ]  UUUUUUU5        g rx   ry   r?   s	           rI   r7   InstanceNorm3D.__init__  r{   rK   c                z    [        UR                  5      S:w  a"  [        S[        UR                  5       S35      eg N   zexpected 5D input (got r~   r   rO   s     rI   rQ   InstanceNorm3D._check_input_dim  r   rK   r   )r_   r`   NNNCDHWN)rA   rb   rB   rc   rC   rc   rD   r   rE   r   rF   zLiteral['NCDHW']rG   re   rf   rg   rh   r   rs   s   @rI   r   r   @  s    J^ /3-1(/

 
 	

 -
 +
 &
 
 

 
( rK   r   c            
         ^  \ rS rSr% SrS\S'   S\S'   \" SS/5       SSS	S	S	S	S
S	S.                     SU 4S jjjj5       rSS jrSS jr	Sr
U =r$ )	GroupNormi  a  
This interface is used to construct a callable object of the ``GroupNorm`` class.
For more details, refer to code examples.
It implements the function of the Group Normalization Layer.
Refer to `Group Normalization <https://arxiv.org/abs/1803.08494>`_ .

Parameters:
    num_groups(int): The number of groups that divided from channels.
    num_channels(int): The number of channels of input.
    epsilon(float, optional): The small value added to the variance to prevent
        division by zero. Default: 1e-05.
        alias: ``eps``.
    affine(bool, optional): Whether this module has learnable affine parameters (weight and bias).
        If set to ``False``, no learnable parameters will be created, regardless of the settings of
        `weight_attr` and `bias_attr`. Defaults to True.
        **Note: This argument must be passed as a keyword argument.**
    device(PlaceLike, optional): Device where the computation takes place. Default: None.
        **Note: This argument must be passed as a keyword argument.**
    dtype(DTypeLike, optional): Data type of the weights and bias. Default: None.
        **Note: This argument must be passed as a keyword argument.**
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for the learnable scale :math:`g`.
        This setting only takes effect when `affine` is ``True``.
        - If set to ``False``, no scale parameter will be created.
        - If set to ``True`` or a `ParamAttr` object, a learnable scale parameter will be created.
          When set to ``True``, it is equivalent to ``ParamAttr()`` with default initialization.
        - If set to ``None``, a learnable scale parameter will be created and initialized to one.
        Default: None.
        **Note: This argument must be passed as a keyword argument.**
    bias_attr (ParamAttr|bool|None, optional): The parameter attribute for the learnable bias :math:`b`.
        This setting only takes effect when `affine` is ``True``.
        - If set to ``False``, no bias parameter will be created.
        - If set to ``True`` or a `ParamAttr` object, a learnable bias parameter will be created.
          When set to ``True``, it is equivalent to ``ParamAttr()`` with default initialization.
        - If set to ``None``, a learnable bias parameter will be created and initialized to zero.
        Default: None.
        **Note: This argument must be passed as a keyword argument.**
    data_format(str, optional): Specify the input data format. Support "NCL", "NCHW", "NCDHW", "NLC", "NHWC" or "NDHWC". Default: "NCHW".
        **Note: This argument must be passed as a keyword argument.**
    name(str|None, optional): Name for the GroupNorm, default is None. For more information, please refer to :ref:`api_guide_Name`.
        **Note: This argument must be passed as a keyword argument.**

Shape:
    - x: Tensor with shape: attr:`(batch, num_features, *)`.
    - output: The same shape as input x.

Returns:
    None

Examples:
    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.arange(48, dtype="float32").reshape((2, 6, 2, 2))
        >>> group_norm = paddle.nn.GroupNorm(num_channels=6, num_groups=6)
        >>> group_norm_out = group_norm(x)

        >>> print(group_norm_out)
        Tensor(shape=[2, 6, 2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]]],
         [[[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]],
          [[-1.34163547, -0.44721183],
           [ 0.44721183,  1.34163547]]]])
r   rT   r.   rB   rU   TNra   )affinedevicedtyperD   rE   rF   rG   c          	       > [         TU ]  5         X0l        X l        Xl        XPl        Uc  U R                  R                  5       OUU l        U	S;  a  [        SU	-   5      eU	S   S:X  a  SOSn	Xl
        U R                  /nU(       d  SnSnXpl        Xl        USL a  S U l        OU R                  U R                  UU R                  [        S5      U R
                  S	9U l        U R                  S L=(       a6    [!        U R                  S
5      =(       a    U R                  R"                  S:H  U R                  l        USL a  S U l        g U R                  U R                  UU R                  [        S5      SU R
                  S9U l        U R                  S L=(       a6    [!        U R                  S
5      =(       a    U R                  R"                  S:H  U R&                  l        g )N)r   ra   r   NLCNHWCNDHWCzunsupported data layout:r   Cra   r   Fr0   )r1   r2   r   r3   r   learning_rater5   T)r1   r2   r   r3   r4   r   )r6   r7   r9   _num_channels_num_groups_device_helperr   _dtyper   r=   r:   r;   rT   r>   r   hasattrr   stop_gradientr.   )r@   
num_groupsnum_channelsrB   r   r   r   rD   rE   rF   rG   param_shaperH   s               rI   r7   GroupNorm.__init__  s    	)%05DLL**,5 	 NN7+EFF +A# 5f6'))*KI'#%DK//&&!kk$,SM|| 0 DK )-(9(9(E )))?; ;%%33s: KK%
 DI--__!kk$,SM|| . DI '+ooT&A '9 9OO11S8 II#rK   c                    [        UU R                  U R                  U R                  U R                  U R
                  5      $ rx   )r   r   r9   rT   r.   r=   rO   s     rI   rV   GroupNorm.forwardG  s9    MMKKII
 	
rK   c                T    SU R                    SU R                   SU R                   3$ )Nznum_groups=z, num_channels=rZ   )r   r   r9   r[   s    rI   r\   GroupNorm.extra_reprQ  s2    T--.od>P>P=QQ[\`\i\i[jkkrK   )
r;   r=   r   r   r9   r   r   r:   r.   rT   r_   )r   rb   r   rb   rB   rc   r   boolr   PlaceLike | Noner   DTypeLike | NonerD   r   rE   r   rF   *DataLayout1D | DataLayout2D | DataLayout3DrG   re   rf   rg   rh   ri   rk   rl   rm   rn   ro   rp   r	   r7   rV   r\   rq   rr   rs   s   @rI   r   r     s    Sj N
Li'(
 	C #'"&/3-1BHCC C 	C C !C  C -C +C @C C 
C C )CJ
l lrK   r   c            
         ^  \ rS rSr% SrS\S'   S\S'   \" SS/5       SSSS	S	S	S	S	S
.                   SU 4S jjjj5       rSS jrSS jr	Sr
U =r$ )	LayerNormiU  aN  
Construct a callable object of the ``LayerNorm`` class.
For more details, refer to code examples.
It implements the function of the Layer Normalization Layer and can be applied to mini-batch input data.
Refer to `Layer Normalization <https://arxiv.org/pdf/1607.06450v1.pdf>`_

The formula is as follows:

..  math::

    \mu & = \frac{1}{H}\sum_{i=1}^{H} x_i

    \sigma & = \sqrt{\frac{1}{H}\sum_{i=1}^{H}{(x_i - \mu)^2} + \epsilon}

    y & = f(\frac{g}{\sigma}(x - \mu) + b)

- :math:`x`: the vector representation of the summed inputs to the neurons in that layer.
- :math:`H`: the number of hidden units in a layers
- :math:`\epsilon`: the small value added to the variance to prevent division by zero.
- :math:`g`: the trainable scale parameter.
- :math:`b`: the trainable bias parameter.

Parameters:
    normalized_shape(int|list|tuple): Input shape from an expected input of
        size ``[*, normalized_shape[0], normalized_shape[1], ..., normalized_shape[-1]]`` .
        If it is a single integer, this module will normalize over the last dimension
        which is expected to be of that specific size.
    epsilon(float, optional): The small value added to the variance to prevent
        division by zero. Default: 1e-05.
        alias: ``eps``.
    elementwise_affine(bool, optional): Whether to apply element-wise affine transformation
        (i.e., learnable scale and bias). If set to ``False``, both the scale (:math:`g`) and
        bias (:math:`b`) parameters will be disabled, regardless of the settings of `weight_attr`
        and `bias_attr`. This parameter acts as a master switch. Defaults to True.
        **Note: This argument must be passed as a keyword argument.**
    bias(bool, optional): Whether to include a learnable bias term in the layer. This setting
        only takes effect when `elementwise_affine` is ``True``. If set to ``False``, no bias
        parameter will be created, even if `bias_attr` is specified. Defaults to True.
        **Note: This argument must be passed as a keyword argument.**
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for the learnable
        gain :math:`g` (scale). This setting only takes effect when `elementwise_affine` is ``True``.
        - If set to ``False``, no gain parameter will be created.
        - If set to ``None`` or ``True``, a default :code:`ParamAttr` will be used, and the
          parameter will be initialized to 1.
        - If set to a custom :code:`ParamAttr` object, it will be used to configure the parameter.
        Default: None.
        **Note: This argument must be passed as a keyword argument.**
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the learnable
        bias :math:`b`. This setting only takes effect when both `elementwise_affine` and `bias` are ``True``.
        - If set to ``False``, no bias parameter will be created.
        - If set to ``None`` or ``True``, a default :code:`ParamAttr` will be used, and the
          parameter will be initialized to 0.
        - If set to a custom :code:`ParamAttr` object, it will be used to configure the parameter.
        Default: None.
        **Note: This argument must be passed as a keyword argument.**
    name(str|None, optional): Name for the LayerNorm, default is None. For more information, please refer to :ref:`api_guide_Name` .
        **Note: This argument must be passed as a keyword argument.**

Shape:
    - x: 2-D, 3-D, 4-D or 5-D tensor.
    - output: same shape as input x.

Returns:
    ``Tensor`` , the dimension is the same as :attr:`x`, but the internal values have been normalized by ``LayerNorm`` .

Examples:

    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.rand((2, 2, 2, 3))
        >>> layer_norm = paddle.nn.LayerNorm(x.shape[1:])
        >>> layer_norm_out = layer_norm(x)

        >>> print(layer_norm_out)
        Tensor(shape=[2, 2, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[[ 0.60520101, -0.67670590, -1.40020895],
           [ 0.46540466, -0.09736638, -0.47771254]],
          [[-0.74365306,  0.63718957, -1.41333175],
           [ 1.44764745, -0.25489068,  1.90842617]]],
         [[[ 1.09773350,  1.49568415, -0.45503747],
           [-1.01755989,  1.08368254, -0.38671425]],
          [[-0.62252408,  0.60490781,  0.13109133],
           [-0.81222653,  0.84285998, -1.96189952]]]])
r,   rT   r.   rB   rU   TN)elementwise_affiner.   r   r   rD   rE   rG   c          	       > [         TU ]  5         [        U[        R                  5      (       a  U/n[        U5      U l        X l        XPl        Uc  U R                  R                  5       OUU l        U(       d  SnSnO	U(       d  SnXpl        Xl        [        R                  " U R                  5      /n
USL a  S U l        O?U R#                  U R                  U R                  U
[%        S5      U R                  S9U l        USL a  S U l        g U R#                  U R                  U R                  U
[%        S5      U R                  SS9U l        g )NFr0   )r1   r   r2   r3   r   r5   T)r1   r   r2   r3   r   r4   )r6   r7   
isinstancenumbersIntegrallist_normalized_shaper9   r   r   r   r   r:   r;   npprodrT   r>   r   r.   )r@   normalized_shaperB   r   r.   r   r   rD   rE   rG   r   rH   s              rI   r7   LayerNorm.__init__  s,    	&(8(899 01!%&6!705DLL**,5 	 "KII'#wwt5567%DK//&&kk!$,SM|| 0 DK DI--__kk!$,SM|| . DIrK   c                l    [        UU R                  U R                  U R                  U R                  S9$ )N)r   rT   r.   rB   )r   r   rT   r.   r9   rO   s     rI   rV   LayerNorm.forward  s0    !33;;MM
 	
rK   c                :    SU R                    SU R                   3$ )Nznormalized_shape=rZ   )r   r9   r[   s    rI   r\   LayerNorm.extra_repr  s     "4#9#9":*T]]OTTrK   )r;   r   r   r9   r   r:   r.   rT   r   )r   zint | Sequence[int]rB   rc   r   r   r.   r   r   r   r   r   rD   r   rE   r   rG   re   rf   rg   rh   ri   r   rs   s   @rI   r   r   U  s    Un 
i'( 7
 $(#'"&/3-17-7 7
 !7 7 !7  7 -7 +7 7 
7 7 )7r
U UrK   r   c                     ^  \ rS rSr% SrS\S'   S\S'          S                 SU 4S jjjrSS jrSS jrSS	 jr	SS
 jr
SrU =r$ )_BatchNormBasei  z
BatchNorm base .
r,   rT   r.   c	           
     ~  > [         TU ]  5         Xl        X@l        XPl        Xpl        [        5       S:X  a  SU l        O[        5       U l        U/n	USLa5  U R                  U R                  U	U R                  [        S5      S9U l
        OS U l
        USLa,  U R                  U R                  U	U R                  SS9U l        OS U l        S n
S nUb
  US-   n
US	-   nU R                  U R                  [        U
[        S
5      SSS9U	S9U l        SU R                  l        U R                  U R                  [        U[        S5      SSS9U	S9U l        SU R                  l        [!        5       S   (       Ga  S[#        5       ;   Ga  [%        5          [&        R(                  " U R                  S5      n[&        R(                  " U R                  S5      n[&        R(                  " U R                  S5      n[&        R(                  " U R                  S5      nUR+                  U R                  5        UR+                  U R                  5        UR+                  U R                  5        UR+                  U R                  5        S S S 5        X`l        SU l        X l        X0l        SU l        Xl        g ! , (       d  f       N5= f)Nfloat16float32Fr0   r1   r2   r   r3   Tr1   r2   r   r4   _mean	_variancer5   rG   initializer	trainabledo_model_average)r   r1   r2   FLAGS_npu_storage_formatnpur
   )r6   r7   r<   r:   r;   _use_global_statsr   r   r>   r   rT   r.   r   r   r   r   r   r   r   r   npu_identity_share_underline_tensor_tor=   	_in_placer8   r9   _fuse_with_relu_name)r@   rA   rC   rB   rD   rE   rF   use_global_statsrG   r   moving_mean_namemoving_variance_nameweight_trans
bias_trans
mean_trans	var_transrH   s                   rI   r7   _BatchNormBase.__init__  s    	)'#!1)+#DK+-DK#n e#//&&!kk$,SM	 0 DK DKE!--__!kk	 . DI DI##g~#'+#5 **++%$SM!%	  + 	

 $(

 ..++)$SM!%	  / 	
 (,$ O677355%22KK  $00IIq
 $00JJ
 #//NNA	 77D55dii@55djjA44T^^D! $ (!$
/ s   C1J..
J<c                    [        S5      e)NzBatchNorm Base errorrM   rO   s     rI   rQ   _BatchNormBase._check_input_dimi  s    !"899rK   c                    [        S5      e)Nz BatchNorm Base data format errorrM   rO   s     rI   _check_data_format!_BatchNormBase._check_data_formatl  s    !"DEErK   c                   U R                  U R                  5        U R                  U5        U R                  (       a  [        R
                  " S5        [        UU R                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  S9
$ )Nz<When training, we now always track global mean and variance.)rT   r.   trainingrC   rB   rF   r   )r   r=   rQ   r   warningswarnr   r   r   rT   r.   r8   r9   r   rO   s     rI   rV   _BatchNormBase.forwardo  s     1 12e$==MMN JJNN;;]]^^MM))!33
 	
rK   c                    SU R                    SU R                   SU R                   3nU R                  S:w  a  USU R                   3-  nU R                  b  USU R                   3-  nU$ )NrY   z, momentum=rZ   ra   , data_format=, name=)r<   r8   r9   r=   r   r@   main_strs     rI   r\   _BatchNormBase.extra_repr  sy    "4#5#5"6k$..AQQ[\`\i\i[jk&.):):(;<<H::!'$**..HrK   )r;   r=   r   r9   r   r   r   r8   r   r<   r   r   r:   r.   rT   )r`   r_   NNra   NN)rA   rb   rC   rc   rB   rc   rD   rd   rE   rd   rF   r$   r   bool | NonerG   re   rf   rg   r   rh   ri   )rk   rl   rm   rn   ro   rp   r7   rQ   r   rV   r\   rq   rr   rs   s   @rI   r   r     s     

 ,0*.$*(,hh h 	h
 *h (h "h &h h 
h hT:F
. rK   r   c                     ^  \ rS rSr% SrS\S'   S\S'                 S	                               S
U 4S jjjrSS jrSrU =r	$ )	BatchNormi  aa  
This interface is used to construct a callable object of the ``BatchNorm`` class.
For more details, refer to code examples.
It implements the function of the Batch Normalization Layer and can be used
as a normalizer function for conv2d and fully connected operations.
The data is normalized by the mean and variance of the channel based on the current batch data.
Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shift <https://arxiv.org/pdf/1502.03167.pdf>`_
for more details.

When use_global_stats = False, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
Calculated as follows:

..  math::

    \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &
    //\ mini-batch\ mean \\
    \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \mu_{\beta})^2 \qquad &
    //\ mini-batch\ variance \\

- :math:`x` : mini-batch data
- :math:`m` : the size of the mini-batch data

When use_global_stats = True, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch.
They are global or running statistics (moving_mean and moving_variance). It usually got from the
pre-trained model. Calculated as follows:

.. math::
    moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global mean \\
    moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global variance \\

The normalization function formula is as follows:

..  math::

    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
    \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift


- :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
- :math:`\gamma` : trainable proportional parameter
- :math:`\beta` : trainable deviation parameter

Parameters:
    num_channels(int): Indicate the number of channels of the input ``Tensor``.
    act(str, optional): Activation to be applied to the output of batch normalization. Default: None.
    is_test (bool, optional): A flag indicating whether it is in test phrase or not.
         This flag only has effect on static graph mode. For dygraph mode, please use ``eval()``.
         Default: False.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
    param_attr(ParamAttr|bool|None, optional): The parameter attribute for Parameter `scale`
         of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
         will create ParamAttr as param_attr. If the Initializer of the param_attr
         is not set, the parameter is initialized with Xavier. Default: None.
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the bias of batch_norm.
         If it is set to None or one attribute of ParamAttr, batch_norm
         will create ParamAttr as bias_attr. If the Initializer of the bias_attr
         is not set, the bias is initialized zero. Default: None.
    dtype(str, optional): Indicate the data type of the input ``Tensor``,
         which can be float32 or float64. Default: float32.
    data_layout(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC", where `N` is batch size, `C` is the number of the feature map, `H` is the height of the feature map, `W` is the width of the feature map. Default: NCHW.
    in_place(bool, optional): Make the input and output of batch norm reuse memory. Default: False.
    moving_mean_name(str, optional): The name of moving_mean which store the global Mean. Default: None.
    moving_variance_name(str, optional): The name of the moving_variance which store the global Variance. Default: None.
    do_model_average_for_mean_and_var(bool, optional): Whether parameter mean and variance should do model
        average when model average is enabled. Default: True.
    use_global_stats(bool, optional): Whether to use global mean and
        variance. In inference or test mode, set use_global_stats to true
        or is_test to true, and the behavior is equivalent.
        In train mode, when setting use_global_stats True, the global mean
        and variance are also used during train period. Default: False.
    trainable_statistics(bool, optional): Whether to calculate mean and var in eval mode. In eval mode, when
        setting trainable_statistics True, mean and variance will be calculated by current batch statistics.
        Default: False.

Returns:
    None

Examples:
    .. code-block:: python

        >>> import paddle.nn as nn
        >>> import paddle
        >>> import numpy as np

        >>> x = paddle.rand(shape=(3, 10, 3, 7), dtype="float32")
        >>> batch_norm = nn.BatchNorm(10)
        >>> hidden1 = batch_norm(x)
r,   rT   r.   c           	       > [         TU ]  5         X`l        Xpl        X l        US:X  a  SU l        OXl        U/nUSLa5  U R                  U R                  UU R
                  [        S5      S9U l        OS U l        USLa,  U R                  U R                  UU R
                  SS9U l	        OS U l	        U R                  [        U[        S5      SUS	9UU R
                  S
9U l        SU R                  l        U R                  [        U[        S5      SUS	9UU R
                  S
9U l        SU R                  l        [        5       S   (       Ga"  S[        5       ;   Ga  [!        5       (       Ga  [#        5          [$        R&                  " U R                  S5      n[$        R&                  " U R                  S5      n[$        R&                  " U R                  S5      n[$        R&                  " U R                  S5      nUR)                  U R                  5        UR)                  U R                  5        UR)                  U R                  5        UR)                  U R                  5        S S S 5        Xl        Xl        X@l        XPl        X0l        SU l        Xl        Xl        U R2                  (       + U l        g ! , (       d  f       NV= f)Nr   r   Fr0   r   Tr   r5   r   )r1   r2   r   r   r   r
   )r6   r7   _param_attrr;   _actr   r>   r   rT   r.   r   r   r   r   r   r   r   r   r   r   r   r   _data_layoutr8   r9   _is_testr   r   _trainable_statisticsr   )r@   r   actis_testrC   rB   
param_attrrE   r   data_layoutin_placer   r   !do_model_average_for_mean_and_varr   trainable_statisticsr   r   r   r   r   rH   s                        rI   r7   BatchNorm.__init__  sv   $ 	%#	I#DKK#n U"//%%!kk$,SM	 0 DK DKE!--__!kk	 . DI DI**%$SM!B	 ++ + 	

 $(

 ..)$SM!B	 ++ / 	
 (,$ O677355  Y#)#6#6Q$L "(!4!4		1"J "(!4!4

A"J !' 3 3!I !;;DKKH99$))D99$**E88H! $ "'!$!1%9" MM)5 Ys   6C1J77
Kc                x   [        5       (       a  [        R                  " UU R                  U R                  U R
                  U R                  U R                  (       + U R                  U R                  U R                  U R                  U R                  5      u  p#pEpgU R                  c  U$ [        R                  " X R                  S9$ [!        5       (       a  [        R"                  " UU R                  U R                  U R
                  U R                  U R                  (       + U R                  U R                  U R                  U R                  U R                  5      u  p#pEpg[$        R&                  " X R                  5      $ U R                  nU R                  n	[)        US/ SQS5        U R                  U R                  U R*                  U R                  U R,                  U R                  U R                  S.n
U/U R
                  /U R                  /U R                  /U R                  /S.nU R.                  R1                  U R2                  SS9nU R.                  R1                  U R2                  SS9nU R.                  R1                  U R.                  R5                  U5      SS9nU R6                  (       a  UO$U R.                  R1                  U R2                  5      nU/U/U	/U/U/S	.nUb  U/US
'   U R.                  R9                  SXU
S9  U R.                  R;                  X R                  5      $ )N)r  rP   )r   r   float64r   rC   rB   r  r  fuse_with_relur   r	  XScaleBiasMeanVarianceTr   r   YMeanOutVarianceOut	SavedMeanSavedVarianceReserveSpacer   typeinputsoutputsattrs)r   r   r   r   r   rT   r.   r   r8   r9   r   r   r  r   r   _append_activation_in_dygraphr   batch_norm_r   append_activation_in_pirr   r  r   r   "create_variable_for_type_inferencer   input_dtyper   	append_opappend_activation)r@   rP   batch_norm_outt1t2t3t4_mean_outvariance_outr!  r  
saved_meansaved_variancereserve_spacer   s                   rI   rV   BatchNorm.forward]  s   060A0A

		MM!!!&&**1-N yy %% >>II  ]]060B0B

		MM!!!&&**1-N 55niiPP zzH>>L$w A;
 !NN====#00"&"6"6$($:$:(,(B(BE W++!^^,F HHkk I J "\\LLkk M N !LLKKll..u5T L M >> \\DDKK  %%$: ,~(\"0!1G (+8/'LL""!& # 
 <<11.))LLrK   )r   r;   r   r   r9   r   r   r  r   r8   r   r  r   r   r.   r   rT   )NFr`   r_   NNr   ra   FNNTFF) r   rb   r  re   r  r   rC   rc   rB   rc   r  rd   rE   rd   r   r%   r  r"   r  r   r   re   r   re   r  r   r   r   r	  r   rf   rg   rh   
rk   rl   rm   rn   ro   rp   r7   rV   rq   rr   rs   s   @rI   r   r     s   \| 

 +/*.$$*'++/26!&%*!j*j* j* 	j*
 j* j* )j* (j* j* "j* j* %j* )j* ,0j* j*  #!j*" 
#j* j*X`M `MrK   r   c                     ^  \ rS rSrSr       S                 SU 4S jjjr    S	S jrS
S jrSrU =r	$ )BatchNorm1Di  a  
Applies Batch Normalization over a 2D or 3D input (a mini-batch of 1D inputs with additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .

When use_global_stats = False, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
Calculated as follows:

..  math::

    \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
    \ mini-batch\ mean \\
    \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
    \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\

- :math:`x` : mini-batch data
- :math:`m` : the size of the mini-batch data

When use_global_stats = True, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch.
They are global or running statistics (moving_mean and moving_variance). It usually got from the
pre-trained model. Calculated as follows:

.. math::
    moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
    moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\

The normalization function formula is as follows:

..  math::

    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift

- :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
- :math:`\gamma` : trainable proportional parameter
- :math:`\beta` : trainable deviation parameter

Parameters:
    num_features(int): Indicate the number of channels of the input ``Tensor``.
    epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for Parameter `scale`
        of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
        will create ParamAttr as weight_attr. If it is set to False, the weight is not learnable.
        If the Initializer of the weight_attr is not set, the parameter is initialized with ones. Default: None.
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the bias of batch_norm.
        If it is set to None or one attribute of ParamAttr, batch_norm
        will create ParamAttr as bias_attr. If it is set to False, the weight is not learnable.
        If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None.
    data_format(str, optional): Specify the input data format, may be "NC", "NCL" or "NLC", where `N` is batch size, `C` is the number of the feature map, `L` is the length of the feature map. Default "NCL".
    use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
    name(str|None, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

Shape:
    - x: 2-D or 3-D tensor with shape: (batch, num_features) or (batch, num_features, length) when data_format is "NC" or "NCL",
        (batch, length, num_features) when data_format is "NLC".
    - output: 3-D tensor with same shape as input x.

Returns:
    None.


Examples:
    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.rand((2, 1, 3))
        >>> batch_norm = paddle.nn.BatchNorm1D(1)
        >>> batch_norm_out = batch_norm(x)

        >>> print(batch_norm_out)
        Tensor(shape=[2, 1, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[ 1.26652932, -0.60229754, -1.65705597]],
         [[ 1.06272745,  0.24229205, -0.31219530]]])
c	           
     2   > [         T	U ]  UUUUUUUU5        g rx   ry   
r@   rA   rC   rB   rD   rE   rF   r   rG   rH   s
            rI   r7   BatchNorm1D.__init__  *     			
rK   c                t    US:X  d  US:X  d  US:X  a  SU l         g US:X  d  US:X  a  SU l         g [        S5      e)Nra   NCr   r   r   z4expected NC , NCL, NLC or None for data_format inputr=   r   rO   s     rI   r   BatchNorm1D._check_data_format$  sD     F?etmu~ &Df_ &DF rK   c                    [        UR                  5      S:w  a<  [        UR                  5      S:w  a"  [        S[        UR                  5       S35      eg g r}   r   rO   s     rI   rQ   BatchNorm1D._check_input_dim0  r   rK   r=   )r`   r_   NNr   NN)rA   rb   rC   rc   rB   rc   rD   rd   rE   rd   rF   r!   r   r   rG   re   rf   rg   )rP   z*DataLayout0D | DataLayout1D | DataLayout2Drf   rg   r   
rk   rl   rm   rn   ro   r7   r   rQ   rq   rr   rs   s   @rI   r7  r7    s    K` ,0*.$)(,

 
 	

 *
 (
 "
 &
 
 

 
,
?
	
 rK   r7  c                  ,    \ rS rSrSrSS jrSS jrSrg)	BatchNorm2Di7  a  
Applies Batch Normalization over a 4D input (a mini-batch of 2D inputs with additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .

When use_global_stats = False, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
Calculated as follows:

..  math::

    \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//
    \ mini-batch\ mean \\
    \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i -
    \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\

When use_global_stats = True, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch.
They are global or running statistics (moving_mean and moving_variance). It usually got from the
pre-trained model. Calculated as follows:

.. math::
    moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
    moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\

The normalization function formula is as follows:

..  math::

    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift

- :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
- :math:`\gamma` : trainable proportional parameter
- :math:`\beta` : trainable deviation parameter

Parameters:
    num_features(int): Indicate the number of channels of the input ``Tensor``.
    epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for Parameter `scale`
        of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
        will create ParamAttr as weight_attr. If it is set to False, the weight is not learnable.
        If the Initializer of the weight_attr is not set, the parameter is initialized with ones. Default: None.
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the bias of batch_norm.
        If it is set to None or one attribute of ParamAttr, batch_norm
        will create ParamAttr as bias_attr. If it is set to False, the weight is not learnable.
        If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None.
    data_format(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC", where `N` is batch size, `C` is the number of the feature map, `H` is the height of the feature map, `W` is the width of the feature map. Default: NCHW.
    use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
    name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

Shape:
    - x: 4-D tensor with shape: (batch, num_features, height, weight) when data_format is "NCHW",
        or (batch, height, weight, num_features) when data_format is "NHWC".
    - output: 4-D tensor with same shape as input x.

Returns:
    None

Examples:
    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.rand((2, 1, 2, 3))
        >>> batch_norm = paddle.nn.BatchNorm2D(1)
        >>> batch_norm_out = batch_norm(x)

        >>> print(batch_norm_out)
        Tensor(shape=[2, 1, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[[ 0.60520101, -0.67670590, -1.40020895],
           [ 0.46540475, -0.09736633, -0.47771257]]],
         [[[-0.74365312,  0.63718963, -1.41333187],
           [ 1.44764757, -0.25489068,  1.90842628]]]])
c                2    US;  a  [        S5      eXl        g )N)ra   r   z+expected NCHW or NHWC for data_format input)r   r=   rO   s     rI   r   BatchNorm2D._check_data_format  s    ((JKK!rK   c                z    [        UR                  5      S:w  a"  [        S[        UR                  5       S35      eg r   r   rO   s     rI   rQ   BatchNorm2D._check_input_dim  r   rK   rB  N)rP   r"   rf   rg   r   )rk   rl   rm   rn   ro   r   rQ   rq   r   rK   rI   rE  rE  7  s    IV"
rK   rE  c                  x   ^  \ rS rSrSr       S                 SU 4S jjjrS	S jrS
S jrSrU =r	$ )BatchNorm3Di  a8  
Applies Batch Normalization over a 5D input (a mini-batch of 3D inputs with additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .

When use_global_stats = False, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
Calculated as follows:

..  math::

    \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
    \ mini-batch\ mean \\
    \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
    \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\

When use_global_stats = True, the :math:`\\mu_{\\beta}`
and :math:`\\sigma_{\\beta}^{2}` are not the statistics of one mini-batch.
They are global or running statistics (moving_mean and moving_variance). It usually got from the
pre-trained model. Calculated as follows:

.. math::
    moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
    moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\

The normalization function formula is as follows:

..  math::

    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift

- :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
- :math:`\gamma` : trainable proportional parameter
- :math:`\beta` : trainable deviation parameter

Parameters:
    num_features(int): Indicate the number of channels of the input ``Tensor``.
    epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for Parameter `scale`
        of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
        will create ParamAttr as weight_attr. If it is set to False, the weight is not learnable.
        If the Initializer of the weight_attr is not set, the parameter is initialized with ones. Default: None.
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the bias of batch_norm.
        If it is set to None or one attribute of ParamAttr, batch_norm
        will create ParamAttr as bias_attr. If it is set to False, the weight is not learnable.
        If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None.
    data_format(str, optional): Specify the input data format, the data format can be "NCDHW" or "NDHWC", where `N` is batch size, `C` is the number of the feature map, `D` is the depth of the feature, `H` is the height of the feature map, `W` is the width of the feature map. Default: NCDHW.
    use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
    name(str|None, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

Shape:
    - x: 5-D tensor with shape: (batch, num_features, dims, height, weight) when data_format is "NCDHW",
        or (batch, dims, height, weight, num_features) when data_format is "NDHWC".
    - output: 5-D tensor with same shape as input x.

Returns:
    None

Examples:
    .. code-block:: python

        >>> import paddle
        >>> paddle.seed(100)
        >>> x = paddle.rand((2, 1, 2, 2, 3))
        >>> batch_norm = paddle.nn.BatchNorm3D(1)
        >>> batch_norm_out = batch_norm(x)

        >>> print(batch_norm_out)
        Tensor(shape=[2, 1, 2, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
        [[[[[ 0.28011751, -0.95211101, -1.64757574],
            [ 0.14573872, -0.39522290, -0.76082933]],
           [[-1.01646376,  0.31086648, -1.66019011],
            [ 1.08991623, -0.54664266,  1.53283834]]]],
         [[[[ 1.33958006,  1.71585774, -0.12862551],
            [-0.66051245,  1.32629418, -0.06402326]],
           [[-0.28699064,  0.87359405,  0.42558217],
            [-0.46636176,  1.09858704, -1.55342245]]]]])
c	           
     2   > [         T	U ]  UUUUUUUU5        g rx   ry   r9  s
            rI   r7   BatchNorm3D.__init__  r;  rK   c                h    US:X  d  US:X  a  SU l         g US:X  d  US:X  a  SU l         g [        S5      e)Nra   r   r   r   z3expected NCDHW, NDHWC or None for data_format inputr>  rO   s     rI   r   BatchNorm3D._check_data_format  s>    F?ew. &Df_ 0 &DE rK   c                z    [        UR                  5      S:w  a"  [        S[        UR                  5       S35      eg r   r   rO   s     rI   rQ   BatchNorm3D._check_input_dim  r   rK   rB  )r`   r_   NNr   NN)rA   rb   rC   rc   rB   rc   rD   rd   rE   rd   rF   r#   r   r   rG   re   rf   rg   )rP   zDataLayout2D | DataLayout3Drf   rg   r   rC  rs   s   @rI   rK  rK    s    Md ,0*.$+(,

 
 	

 *
 (
 "
 &
 
 

 
, rK   rK  c                     ^  \ rS rSrSr      S               S	U 4S jjjrS
S jrSS jr\SS j5       r	Sr
U =r$ )SyncBatchNormi  a  

This interface is used to construct a callable object of the ``SyncBatchNorm`` class.
It implements the function of the Cross-GPU Synchronized Batch Normalization Layer, and can
be used as a normalizer function for other operations, such as conv2d and fully connected
operations.
The data is normalized by the mean and variance of the channel based on whole mini-batch
, which including data in all gpus.
Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shift <https://arxiv.org/pdf/1502.03167.pdf>`_
for more details.

When model in training mode, the :math:`\\mu_{\\beta}`
and :math:`\\sigma_{\\beta}^{2}` are the statistics of whole mini-batch data in all gpus.
Calculated as follows:

..  math::

    \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
    \ mini-batch\ mean \\
    \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
    \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\

- :math:`x` : whole mini-batch data in all gpus
- :math:`m` : the size of the whole mini-batch data

When model in evaluation mode, the :math:`\\mu_{\\beta}`
and :math:`\sigma_{\beta}^{2}` are global statistics (moving_mean and moving_variance,
which usually got from the pre-trained model). Global statistics calculated as follows:

.. math::
    moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
    moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\

The formula of normalization is as follows:

..  math::

    \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
    \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
    y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift

- :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
- :math:`\gamma` : trainable scale parameter vector
- :math:`\beta` : trainable shift parameter vector

Note:
    If you want to use container to pack your model and has :ref:`api_paddle_nn_SyncBatchNorm` in the
    evaluation phase, please use :ref:`api_paddle_nn_LayerList` or :ref:`api_paddle_nn_Sequential` instead of
    :ref:`api_paddle_hub_list` to pack the model.

Parameters:
    num_features(int): Indicate the number of channels of the input ``Tensor``.
    epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
    momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
    weight_attr(ParamAttr|bool|None, optional): The parameter attribute for Parameter `scale`
         of this layer. If it is set to None or one attribute of ParamAttr, this layer
         will create ParamAttr as param_attr. If the Initializer of the param_attr
         is not set, the parameter is initialized with ones. If it is set to False,
         this layer will not have trainable scale parameter. Default: None.
    bias_attr(ParamAttr|bool|None, optional): The parameter attribute for the bias of this layer.
         If it is set to None or one attribute of ParamAttr, this layer
         will create ParamAttr as bias_attr. If the Initializer of the bias_attr
         is not set, the bias is initialized zero. If it is set to False, this layer will not
         have trainable bias parameter. Default: None.
    data_format(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC", where `N` is batch size, `C` is the number of the feature map, `H` is the height of the feature map, `W` is the width of the feature map. Default: NCHW.
    name(str|None, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

Shapes:
    - input: Tensor that the dimension from 2 to 5.
    - output: Tensor with the same shape as input.

Examples:
    .. code-block:: python

        >>> # doctest: +REQUIRES(env:GPU)

        >>> import paddle
        >>> import paddle.nn as nn
        >>> paddle.device.set_device('gpu')
        >>> x = paddle.to_tensor([[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]]).astype('float32')

        >>> if paddle.is_compiled_with_cuda():
        ...     sync_batch_norm = nn.SyncBatchNorm(2)
        ...     hidden1 = sync_batch_norm(x)
        ...     print(hidden1)
        Tensor(shape=[1, 2, 2, 2], dtype=float32, place=Place(gpu:0), stop_gradient=False,
        [[[[ 0.26824948,  1.09363246],
           [ 0.26824948, -1.63013160]],
          [[ 0.80956620, -0.66528702],
           [-1.27446556,  1.13018656]]]])

c           
     <  > [         T	U ]  UUUUUUS U5        U/nUSL a;  U R                  S UU R                  [	        S5      S9U l        SU R
                  l        USL a=  U R                  S UU R                  [	        S5      SS9U l        SU R                  l        g g )NFr0   r   Tr5   )r1   r2   r   r3   r4   )r6   r7   r>   r   r   rT   r   r.   )
r@   rA   rC   rB   rD   rE   rF   rG   r   rH   s
            rI   r7   SyncBatchNorm.__init__e  s     			
 $n%//!kk$,SM	 0 DK )-DKK%--!kk$,SM . DI '+DII# rK   c                x    U R                   S;   a  SU l         g U R                   S;   a  SU l         g [        S5      e)N)ra   r   r=  r   ra   )r   r   r   r   zMexpected 'NCDHW', 'NDHWC', 'NCL', 'NLC', 'NC', 'NCHW', 'NHWC' for data_formatr>  r[   s    rI   r    SyncBatchNorm._check_data_format  s@     >> &D":: &Dm rK   c                   U R                  5         U R                  nU R                  n[        5       (       a~  [        R
                  " UU R                  U R                  U R                  U R                  U R                  (       + U R                  U R                  U R                  SS5      u  n        nU$ [        US/ SQS5        U R                  U R                  U R                  (       + U R                  SSSS.nU/U R                  /U R                  /U R                  /U R                  /S.nU R                  R                  U R                  SS9nU R                  R                  U R                  SS9n	U R                  R                  U R                  5      nU/U/U/U/U	/S	.n
U R                  R!                  S
XzUS9  U$ )NFrP   )r   uint16r   r  rS  r  r  Tr  r  sync_batch_normr  )r   r   r   r   r   sync_batch_norm_rT   r.   r   r8   r9   r=   r   r   r%  r   r'  )r@   xr/  r0  sync_batch_norm_outr.  r!  r  r1  r2  r   s              rI   rV   SyncBatchNorm.forward  s   ! ::~~ "##171H1H

		MM!!!2.Aq!Q '& 7		
 }}==(,,# %$)
 kk]YYKZZL(
 \\DD++T E 

 HH++T I 
 #llMMKK

 && z(>$,-
 	"6% 	 	
 #"rK   c           	        Un[        U[        5      (       Ga  UR                  b^  [        UR                  [        5      (       d?  UR                  R                  b(  UR                  R                  S-   UR                  l        UR
                  b^  [        UR
                  [        5      (       d?  UR
                  R                  b(  UR
                  R                  S-   UR
                  l        [        UR                  UR                  UR                  UR                  UR
                  UR                  UR                  5      nUR                  SLaD  UR
                  SLa5  [        5          UR                  Ul        UR                  Ul        SSS5        UR                  Ul        UR                   Ul        UR#                  5        H%  u  p4UR%                  X0R'                  U5      5        M'     AU$ ! , (       d  f       Nl= f)a  
Helper function to convert :class: `paddle.nn.BatchNorm*d` layers in the model to :class: `paddle.nn.SyncBatchNorm` layers.

Parameters:
    layer(Layer): model containing one or more `BatchNorm*d` layers.

Returns:
    The original model with converted SyncBatchNorm layers. If BatchNorm*d layer in the model, use SyncBatchNorm layer instead.

Examples:
    .. code-block:: python

        >>> import paddle
        >>> import paddle.nn as nn

        >>> model = nn.Sequential(nn.Conv2D(3, 5, 3), nn.BatchNorm2D(5))
        >>> sync_model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
        >>> print(sync_model)
        Sequential(
            (0): Conv2D(3, 5, kernel_size=[3, 3], data_format=NCHW)
            (1): SyncBatchNorm(num_features=5, momentum=0.9, epsilon=1e-05)
        )

N_syncF)r   r   r:   r   rG   r;   rS  r<   r8   r9   r=   r   r   rT   r.   r   r   named_childrenadd_sublayerconvert_sync_batchnorm)clslayerlayer_outputrG   sublayers        rI   rc  $SyncBatchNorm.convert_sync_batchnorm  s   4 e^,,""."5#5#5t<<&&++7*/*<*<*A*AG*K""'  ,"5#3#3T::$$))5(-(8(8(=(=(G  %(##""  ""L ""%/$$E1Y*/,,L'(-

L%  "'L%*__L"#224ND%%00: 5  Ys   .#G77
H)r=   r.   rT   )r`   r_   NNra   N)rA   rb   rC   rc   rB   rc   rD   rd   rE   rd   rF   r$   rG   re   rf   rg   )rf   rg   r\  r   rf   r   )re  r   rf   r   )rk   rl   rm   rn   ro   r7   r   rV   classmethodrc  rq   rr   rs   s   @rI   rS  rS    s    \B ,0*.$*%+%+ %+ 	%+
 *%+ (%+ "%+ %+ 
%+ %+NH#T A ArK   rS  c                     ^  \ rS rSr% SrS\S'   S\S'   S\S'   S\S'   S	\S
'   S\S'        S             SU 4S jjjrSS jrSS jrSr	U =r
$ )LocalResponseNormi%  a  
Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions.
For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks <https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf>`_

See more details in :ref:`api_paddle_nn_functional_local_response_norm` .

Parameters:
    size (int): The number of channels to sum over.
    alpha (float, optional): The scaling parameter, positive. Default:1e-4
    beta (float, optional): The exponent, positive. Default:0.75
    k (float, optional): An offset, positive. Default: 1.0
    data_format (str, optional): Specify the data format of the input, and the data format of the output
        will be consistent with that of the input. An optional string from:
        If input is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`,
        the data is stored in the order of: `[batch_size, input_channels, feature_length]`.
        If input is 4-D Tensor, the string could be  `"NCHW"`, `"NHWC"`. When it is `"NCHW"`,
        the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`.
        If input is 5-D Tensor, the string could be  `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`,
        the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`.
    name (str|None, optional): Name for the operation (optional, default is None). For more information,
        please refer to :ref:`api_guide_Name`.

Shape:
    - input: 3-D/4-D/5-D tensor.
    - output: 3-D/4-D/5-D tensor, the same shape as input.

Examples:

    .. code-block:: pycon

        >>> import paddle

        >>> x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32")
        >>> m = paddle.nn.LocalResponseNorm(size=5)
        >>> y = m(x)
        >>> print(y.shape)
        paddle.Size([3, 3, 112, 112])
rb   sizerc   alphabetakr   rF   re   rG   c                j   > [         TU ]  5         Xl        X l        X0l        X@l        XPl        X`l        g rx   )r6   r7   rm  rn  ro  rp  rF   rG   )r@   rm  rn  ro  rp  rF   rG   rH   s          rI   r7   LocalResponseNorm.__init__T  s/     		
	&	rK   c           	         [         R                  " UU R                  U R                  U R                  U R
                  U R                  U R                  5      nU$ rx   )Flocal_response_normrm  rn  ro  rp  rF   rG   )r@   rP   outs      rI   rV   LocalResponseNorm.forwarde  sH    ##IIJJIIFFII
 
rK   c                    SU R                    SU R                   SU R                   SU R                   3nU R                  S:w  a  USU R                   3-  nU R
                  b  USU R
                   3-  nU$ )Nzsize=z, alpha=z, beta=z, k=ra   r   r   )rm  rn  ro  rp  rF   rG   r   s     rI   r\   LocalResponseNorm.extra_reprq  s}    499+Xdjj\4PTPVPVxXv%.)9)9(:;;H99 '$))--HrK   )rn  ro  rF   rp  rG   rm  )g-C6?g      ?r0   ra   N)rm  rb   rn  rc   ro  rc   rp  rc   rF   r   rG   re   rf   rg   rh   ri   )rk   rl   rm   rn   ro   rp   r7   rV   r\   rq   rr   rs   s   @rI   rl  rl  %  s    %N IL
KH;;

 BH  	
  @  
 "
 rK   rl  c                  r   ^  \ rS rSr% SrS\S'   S\S'       S	           S
U 4S jjjrSS jrSrU =r	$ )SpectralNormiz  a`  
This interface is used to construct a callable object of the ``SpectralNorm`` class.
For more details, refer to code examples. It implements the function of the Spectral Normalization Layer.
This layer calculates the spectral normalization value of weight parameters of
fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D
Parameters. Calculations are showed as follows.

Step 1:
Generate vector U in shape of [H], and V in shape of [W].
While H is the :attr:`dim` th dimension of the input weights,
and W is the product result of remaining dimensions.

Step 2:
:attr:`power_iters` should be a positive integer, do following
calculations with U and V for :attr:`power_iters` rounds.

.. math::

    \mathbf{v} := \frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2}

    \mathbf{u} := \frac{\mathbf{W}^{T} \mathbf{v}}{\|\mathbf{W}^{T} \mathbf{v}\|_2}

Step 3:
Calculate :math:`\sigma(\mathbf{W})` and normalize weight values.

.. math::

    \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v}

    \mathbf{W} = \frac{\mathbf{W}}{\sigma(\mathbf{W})}


Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .

Parameters:
    weight_shape(list or tuple): The shape of weight parameter.
    dim(int, optional): The index of dimension which should be permuted to the first before reshaping Input(Weight) to matrix, it should be set as 0 if Input(Weight) is the weight of fc layer, and should be set as 1 if Input(Weight) is the weight of conv layer. Default: 0.
    power_iters(int, optional): The number of power iterations to calculate spectral norm. Default: 1.
    eps(float, optional): The epsilon for numerical stability in calculating norms. Default: 1e-12.
    dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".

Returns:
    None

Examples:
    .. code-block:: pycon

        >>> import paddle
        >>> x = paddle.rand((2, 8, 32, 32))
        >>> spectral_norm = paddle.nn.SpectralNorm(x.shape, dim=1, power_iters=2)
        >>> spectral_norm_out = spectral_norm(x)
        >>> print(spectral_norm_out.shape)
        paddle.Size([2, 8, 32, 32])

r   weight_uweight_vc           	       > [         TU ]  5         X0l        X@l        X l        XPl        [        U5      U l        [        R                  " U R                  5      S:  d   S5       eU[        U R                  5      :  d
   SU 35       eU R                  U R                     n[        R                  " U R                  5      U-  nU R                  [        5       U/U R
                  [        SS5      S9U l        SU R                  l        U R                  [        5       U/U R
                  [        SS5      S9U l        SU R                   l        g )Nr   z5Any dimension of `weight_shape` cannot be equal to 0.zSThe input `dim` should be less than the length of `weight_shape`, but received dim=r5   r0   r   T)r6   r7   _power_iters_eps_dimr   r   _weight_shaper   r   r   r>   r   r   r|  r   r}  )	r@   weight_shapedimpower_itersrU   r   hwrH   s	           rI   r7   SpectralNorm.__init__  s9    	'		!,/wwt))*Q. 	
C	
. S++,, 	
:e	
,
 tyy)GGD&&'1,--#++ &sC 0	 . 
 '+#--#++ &sC 0	 . 
 '+#rK   c           	        Un[        5       (       aM  [        R                  " UU R                  U R                  U R
                  U R                  U R                  5      $ [        USSS/S5        X R                  U R                  S.nU R                  R                  U R                  5      nU R                  R                  SUSU0U R
                  U R                  U R                  S.S	9  U$ )
NrT   r   r  r{  )WeightUVspectral_normOut)r  r  rU   r  )r   r   r  r|  r}  r  r  r  r   r   r%  r   r'  )r@   r\  rT   r  rv  s        rI   rV   SpectralNorm.forward  s    !##''		!!		  	!Hy)4n	
 #T]]Kll==dkkJ s yy#00yy 	 	
 
rK   )r  r   r  r  r  r|  r}  )r   r   g-q=r   )r  r(   r  rb   r  rb   rU   rc   r   r%   rf   rg   ri  r5  rs   s   @rI   r{  r{  z  st    6p 
 $(+(+ (+ 	(+
 (+ (+ 
(+ (+T rK   r{  )D
__future__r   r   r   typingr   numpyr   paddler   r   r   paddle.devicer   paddle.utils.decorator_utilsr	   baser   base.data_feederr   	frameworkr   r   r   r   r   r    r   rt  r   r   r   r   r   r   r   layersr   collections.abcr   r   r   paddle._typingr    r!   r"   r#   r$   r%   r&   r'   r(   __all__r*   ru   r   r   r   r   r   r   r7  rE  rK  rS  rl  r{  r   rK   rI   <module>r     sA   #      5 5 4 8 ! 8   J J * (
 
 
 BM BMJY& Yx]& ]@e& ePjl jlZ_U _UDUU UpnM nMb	t. tnU. Upt. tn\N \~R RjD5 DrK   