
    RЦiM                        S r SSKrSSKJr  SSKrSSKJr  SSKJs  Jr	  SSK
JrJrJrJrJrJrJrJr   SSKJr   " S S\R*                  5      r " S	 S
\R*                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      rS\R6                  S\4S jrS\R6                  S\R6                  S\R6                  S\4S jrS\R6                  S\R6                  S\R6                  S\4S jr  " S S\R.                  5      r! " S S\RD                  5      r# " S  S!\RD                  5      r$ " S" S#\RD                  5      r% " S$ S%\RD                  5      r& " S& S'\RD                  5      r' " S( S)\RD                  5      r( " S* S+\RD                  5      r) " S, S-\RD                  5      r*g! \ a
    SSK
Jr   GNf = f).zNormalization layers and wrappers

Norm layer definitions that support fast norm and consistent channel arg order (always first arg).

Hacked together by / Copyright 2022 Ross Wightman
    N)Tuple   )is_fast_normfast_group_normfast_layer_normfast_rms_norm
rms_norm2dfast_rms_norm2dfast_simple_normsimple_norm)rms_normc            	          ^  \ rS rSr% \R
                  R                  \   \S'      S
S\	S\	S\
S\4U 4S jjjrS rS	rU =r$ )	GroupNorm   
_fast_normnum_channels
num_groupsepsaffinec                 J   > [         TU ]  " X!4X4S.UD6  [        5       U l        g )N)r   r   super__init__r   r   )selfr   r   r   r   kwargs	__class__s         O/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/norm.pyr   GroupNorm.__init__"   s%     	TsTVT&.    c                    U R                   (       a6  [        XR                  U R                  U R                  U R
                  5      $ [        R                  " XR                  U R                  U R                  U R
                  5      $ Nr   r   r   weightbiasr   F
group_normr   xs     r   forwardGroupNorm.forward.   O    ??"1oot{{DIItxxXX<<??DKKDHHUUr   r   )    gh㈵>T)__name__
__module____qualname____firstlineno__torchjitFinalbool__annotations__intfloatr   r)   __static_attributes____classcell__r   s   @r   r   r      s]    		%%
 !
)
) 
) 	
)
 
) 
)V Vr   r   c                      ^  \ rS rSr% Sr\R                  R                  \   \	S'   S\
4U 4S jjrS\R                  S\R                  4S jrS	rU =r$ )

GroupNorm15   zCGroup Normalization with 1 group.
Input: tensor in shape [B, C, *]
r   r   c                 H   > [         TU ]  " SU40 UD6  [        5       U l        g )Nr   r   )r   r   r   r   s      r   r   GroupNorm1.__init__;   s     L3F3&.r   r(   returnc                    U R                   (       a6  [        XR                  U R                  U R                  U R
                  5      $ [        R                  " XR                  U R                  U R                  U R
                  5      $ r!   r"   r'   s     r   r)   GroupNorm1.forward?   r+   r   r,   )r.   r/   r0   r1   __doc__r2   r3   r4   r5   r6   r7   r   Tensorr)   r9   r:   r;   s   @r   r=   r=   5   sM     		%%)S )V V%,, V Vr   r=   c                      ^  \ rS rSr% Sr\R                  R                  \   \	S'     SS\
S\S\4U 4S jjjrS\R                  S	\R                  4S
 jrSrU =r$ )	LayerNormF   z"LayerNorm w/ fast norm option
    r   r   r   r   c                 J   > [         TU ]  " U4X#S.UD6  [        5       U l        g N)r   elementwise_affiner   r   r   r   r   r   r   s        r   r   LayerNorm.__init__K   %     	T3TVT&.r   r(   rA   c                    U R                   (       a8  [        XR                  U R                  U R                  U R
                  5      nU$ [        R                  " XR                  U R                  U R                  U R
                  5      nU$ r!   )r   r   normalized_shaper#   r$   r   r%   
layer_normr'   s     r   r)   LayerNorm.forwardU   sb    ??#8#8$++tyyRVRZRZ[A  Q 5 5t{{DIItxxXAr   r,   ư>Tr.   r/   r0   r1   rD   r2   r3   r4   r5   r6   r7   r8   r   rE   r)   r9   r:   r;   s   @r   rG   rG   F   si    		%%
 	)) ) 	) ) %,,  r   rG   c                   ~   ^  \ rS rSrSr  SS\S\S\4U 4S jjjrS\	R                  S\	R                  4S	 jrS
rU =r$ )LayerNormFp32]   zLayerNorm
    r   r   r   c                 ,   > [         TU ]  " U4X#S.UD6  g rJ   r   r   rL   s        r   r   LayerNormFp32.__init__a        	T3TVTr   r(   rA   c                 R   U R                   b  U R                   R                  5       OS nU R                  b  U R                  R                  5       OS n[        R                  " UR                  5       U R
                  X#U R                  5      R                  UR                  5      nU$ r!   )	r#   r8   r$   r%   rQ   rP   r   todtyper   r(   r#   r$   s       r   r)   LayerNormFp32.forwardj   su    (,(?""$T$(II$9tyy tLLD$9$96RUUVWV]V]^r    rS   r.   r/   r0   r1   rD   r7   r8   r5   r   r2   rE   r)   r9   r:   r;   s   @r   rW   rW   ]   s[     	UU U 	U U %,,  r   rW   c                      ^  \ rS rSr% Sr\R                  R                  \   \	S'     SS\
S\S\4U 4S jjjrS\R                  S	\R                  4S
 jrSrU =r$ )LayerNorm2dq   4LayerNorm for channels of '2D' spatial NCHW tensors r   r   r   r   c                 J   > [         TU ]  " U4X#S.UD6  [        5       U l        g rJ   r   rL   s        r   r   LayerNorm2d.__init__u   rN   r   r(   rA   c                 h   UR                  SSSS5      nU R                  (       a7  [        XR                  U R                  U R
                  U R                  5      nOA[        R                  " XR                  U R                  U R
                  U R                  5      nUR                  SSSS5      nU$ Nr         r   )	permuter   r   rP   r#   r$   r   r%   rQ   r'   s     r   r)   LayerNorm2d.forward   s    IIaAq!??#8#8$++tyyRVRZRZ[AQ 5 5t{{DIItxxXAIIaAq!r   r,   rS   rU   r;   s   @r   re   re   q   sg    ?		%%
 	)) ) 	) ) %,,  r   re   c                   ~   ^  \ rS rSrSr  SS\S\S\4U 4S jjjrS\	R                  S\	R                  4S	 jrS
rU =r$ )LayerNorm2dFp32   rg   r   r   r   c                 ,   > [         TU ]  " U4X#S.UD6  g rJ   rZ   rL   s        r   r   LayerNorm2dFp32.__init__   r\   r   r(   rA   c                    UR                  SSSS5      nU R                  b  U R                  R                  5       OS nU R                  b  U R                  R                  5       OS n[        R
                  " UR                  5       U R                  X#U R                  5      R                  UR                  5      nUR                  SSSS5      nU$ rk   )
rn   r#   r8   r$   r%   rQ   rP   r   r^   r_   r`   s       r   r)   LayerNorm2dFp32.forward   s    IIaAq!(,(?""$T$(II$9tyy tLLD$9$96RUUVWV]V]^IIaAq!r   rb   rS   rc   r;   s   @r   rq   rq      sY    ?
 	UU U 	U U %,,  r   rq   tensorrA   c                     [         R                  R                  5       (       a  U R                  5       $ U R                  [         R                  S9$ )N)memory_format)r2   r3   is_scriptingis_contiguouscontiguous_format)rw   s    r   _is_contiguousr}      s>    yy##%%##%2I2I#JJr   r(   r#   r$   r   c                     [         R                  " U SSSS9u  pEX-
  [         R                  " XC-   5      -  n XS S 2S S 4   -  US S 2S S 4   -   n U $ )Nr   FT)dimunbiasedkeepdim)r2   var_meanrsqrt)r(   r#   r$   r   sus         r   _layer_norm_cfr      sW    >>!UDADA	
%++ag&&A	1dD=!!DD$$77AHr   c                    U R                  SSS9nX -  R                  SSS9XD-  -
  R                  S5      nX-
  [        R                  " XS-   5      -  n XR	                  SSSS5      -  UR	                  SSSS5      -   n U $ )Nr   T)r   r   r   )meanclampr2   r   view)r(   r#   r$   r   r   r   s         r   _layer_norm_cf_sqmr      s    	1d#A
%!T	*ae	4;;A>A	
%++ag&&A	KK2q!$$tyyB1'==AHr   c                   ^   ^  \ rS rSrSrS	S\S\4U 4S jjjrS\R                  4S jr
SrU =r$ )
LayerNormExp2d   aN  LayerNorm for channels_first tensors with 2d spatial dimensions (ie N, C, H, W).

Experimental implementation w/ manual norm for tensors non-contiguous tensors.

This improves throughput in some scenarios (tested on Ampere GPU), esp w/ channels_last
layout. However, benefits are not always clear and can perform worse on other GPUs.
r   r   c                     > [         TU ]  XS9  g )N)r   rZ   )r   r   r   r   s      r   r   LayerNormExp2d.__init__   s    /r   rA   c                 L   [        U5      (       ah  [        R                  " UR                  SSSS5      U R                  U R
                  U R                  U R                  5      R                  SSSS5      nU$ [        XR
                  U R                  U R                  5      nU$ rk   )	r}   r%   rQ   rn   rP   r#   r$   r   r   r'   s     r   r)   LayerNormExp2d.forward   s    !		!Q1%t'<'<dkk499VZV^V^``g`ghiklnoqr`s   q++tyy$((CAr   rb   )rT   )r.   r/   r0   r1   rD   r7   r8   r   r2   rE   r)   r9   r:   r;   s   @r   r   r      s5    0S 0u 0 0ELL  r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'   \
\S'       SS
\S\	S\
SS	4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )RmsNorm   -RmsNorm w/ fast (apex) norm if available
    rP   r   rK   r   .rP   r   rK   r   Nchannelsr   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        [        5       U l
        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R!                  SS 5        U R#                  5         g N)devicer_   r#   r   r   
isinstancenumbersIntegraltuplerP   r   rK   r   r   nn	Parameterr2   emptyr#   register_parameterreset_parameters	r   r   r   r   r   r_   ddrP   r   s	           r   r   RmsNorm.__init__        /#&(8(899 02 %&6 7"(&."",,u{{43H3H'OB'OPDK##Hd3r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   rK   r   initones_r#   r   s    r   r   RmsNorm.reset_parameters   $    ""GGMM$++& #r   r(   c                     U R                   (       a-  [        XR                  U R                  U R                  5      nU$ [        XR                  U R                  U R                  5      nU$ r!   )r   r   rP   r#   r   r   r'   s     r   r)   RmsNorm.forward   sQ     ??a!6!6TXXNA  114;;IAr   r   rK   r   rP   r#   rT   TNNrA   Nr.   r/   r0   r1   rD   __constants__r   r7   r6   r8   r5   r   r   r2   rE   r)   r9   r:   r;   s   @r   r   r          SMCHo%	J
      	  
   4' %,,  r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'       SS	\S\	S
\
SS4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )RmsNormFp32   r   rP   r   rK   .rP   r   rK   Nr   r   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R                  SS 5        U R                  5         g r   r   r   r   r   r   r   rP   r   rK   r   r   r2   r   r#   r   r   r   s	           r   r   RmsNormFp32.__init__       /#&(8(899 02 %&6 7"("",,u{{43H3H'OB'OPDK##Hd3r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   r   r   s    r   r   RmsNormFp32.reset_parameters  r   r   r(   c                     U R                   b  U R                   R                  5       OS n[        UR                  5       U R                  X R                  5      R                  UR                  5      nU$ r!   )r#   r8   r   rP   r   r^   r_   r   r(   r#   s      r   r)   RmsNormFp32.forward   sQ    (,(?""$TQWWY 5 5vxxHKKAGGTr   rK   r   rP   r#   r   r   r   r;   s   @r   r   r          EMCHo%	J
      	  
   2' %,,  r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'   \
\S'       SS
\S\	S\
SS	4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )	RmsNorm2di&  1  RmsNorm2D for NCHW tensors, w/ fast apex or cast norm if available

NOTE: It's currently (2025-05-10) faster to use an eager 2d kernel that does reduction
on dim=1 than to permute and use internal PyTorch F.rms_norm, this may change if something
like https://github.com/pytorch/pytorch/pull/150576 lands.
r   .rP   r   rK   r   Nr   r   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        [        5       U l
        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R!                  SS 5        U R#                  5         g r   r   r   s	           r   r   RmsNorm2d.__init__3  r   r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   r   r   s    r   r   RmsNorm2d.reset_parametersM  r   r   r(   c                     U R                   (       a-  [        XR                  U R                  U R                  5      nU$ [        XR                  U R                  U R                  5      nU$ r!   )r   r
   rP   r#   r   r	   r'   s     r   r)   RmsNorm2d.forwardQ  sQ     ??#8#8$++txxPA  133T[[$((KAr   r   r   r   r   r;   s   @r   r   r   &  s     TMCHo%	J
      	  
   4' %,,  r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'       SS	\S\	S
\
SS4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )RmsNorm2dFp32i[  r   r   .rP   r   rK   Nr   r   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R                  SS 5        U R                  5         g r   r   r   s	           r   r   RmsNorm2dFp32.__init__g  r   r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   r   r   s    r   r   RmsNorm2dFp32.reset_parameters  r   r   r(   c                     U R                   b  U R                   R                  5       OS n[        UR                  5       U R                  X R                  5      R                  UR                  5      nU$ r!   )r#   r8   r	   rP   r   r^   r_   r   s      r   r)   RmsNorm2dFp32.forward  sQ    (,(?""$Tqwwy$"7"7JMMaggVr   r   r   r   r   r;   s   @r   r   r   [  s     FMCHo%	J
      	  
   2' %,,  r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'   \
\S'       SS
\S\	S\
SS	4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )
SimpleNormi  SimpleNorm (x / std(x))
    r   .rP   r   rK   r   Nr   r   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        [        5       U l
        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R!                  SS 5        U R#                  5         g r   r   r   s	           r   r   SimpleNorm.__init__  r   r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   r   r   s    r   r   SimpleNorm.reset_parameters  r   r   r(   c                     U R                   (       a-  [        XR                  U R                  U R                  5      nU$ [        XR                  U R                  U R                  5      nU$ r!   )r   r   rP   r#   r   r   r'   s     r   r)   SimpleNorm.forward  sO    ?? $9$94;;QA  A44dkk488LAr   r   r   r   r   r;   s   @r   r   r     s    SMCHo%	J
      	  
   4' %,,  r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'       SS	\S\	S
\
SS4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )SimpleNormFp32i  r   r   .rP   r   rK   Nr   r   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R                  SS 5        U R                  5         g r   r   r   s	           r   r   SimpleNormFp32.__init__  r   r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   r   r   s    r   r   SimpleNormFp32.reset_parameters  r   r   r(   c                     U R                   b  U R                   R                  5       OS n[        UR                  5       U R                  X R                  5      R                  UR                  5      nU$ r!   )r#   r8   r   rP   r   r^   r_   r   s      r   r)   SimpleNormFp32.forward  sQ    (,(?""$T	4#8#8&((KNNqwwWr   r   r   r   r   r;   s   @r   r   r     r   r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'   \
\S'       SS
\S\	S\
SS	4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )SimpleNorm2di   SimpleNorm for NCHW tensors
    r   .rP   r   rK   r   Nr   r   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        [        5       U l
        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R!                  SS 5        U R#                  5         g r   r   r   s	           r   r   SimpleNorm2d.__init__  r   r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   r   r   s    r   r   SimpleNorm2d.reset_parameters  r   r   r(   c                 &   UR                  SSSS5      nU R                  (       a,  [        XR                  U R                  U R
                  5      nO+[        XR                  U R                  U R
                  5      nUR                  SSSS5      nU$ rk   )rn   r   r   rP   r#   r   r   r'   s     r   r)   SimpleNorm2d.forward  sn    IIaAq!?? $9$94;;QAA44dkk488LAIIaAq!r   r   r   r   r   r;   s   @r   r   r     r   r   r   c            	          ^  \ rS rSr% Sr/ SQr\\S4   \S'   \	\S'   \
\S'       SS	\S\	S
\
SS4U 4S jjjrSS jrS\R                  S\R                  4S jrSrU =r$ )SimpleNorm2dFp32i  r   r   .rP   r   rK   Nr   r   rA   c                   > XES.n[         TU ]  5         Un[        U[        R                  5      (       a  U4n[        U5      U l        X l        X0l        U R                  (       a;  [        R                  " [        R                  " U R                  40 UD65      U l        OU R                  SS 5        U R                  5         g r   r   r   s	           r   r   SimpleNorm2dFp32.__init__  r   r   c                 z    U R                   (       a*  [        R                  R                  U R                  5        g g r!   r   r   s    r   r   !SimpleNorm2dFp32.reset_parameters6  r   r   r(   c                 8   UR                  SSSS5      nU R                  b  U R                  R                  5       OS n[        UR                  5       U R                  X R
                  5      R                  UR                  5      nUR                  SSSS5      nU$ rk   )rn   r#   r8   r   rP   r   r^   r_   r   s      r   r)   SimpleNorm2dFp32.forward:  sy    IIaAq!(,(?""$T	4#8#8&((KNNqwwWIIaAq!r   r   r   r   r   r;   s   @r   r   r     s    EMCHo%	J
      	  
   2' %,,  r   r   )+rD   r   typingr   r2   torch.nnr   torch.nn.functional
functionalr%   	fast_normr   r   r   r   r	   r
   r   r   r   ImportErrorr   r=   rG   rW   re   rq   rE   r5   r}   r8   r   r   r   Moduler   r   r   r   r   r   r   r   rb   r   r   <module>r     s        	 	 	$,
V V,V V" .BLL (",, 0bll *K5<< KD Kell ELL  SX %,,  ELL W\ R\\ *.bii .b(")) (V2		 2j,BII ,^, ,^(RYY (V.299 .b*ryy *u  $##$s   G   G0/G0