
    RЦipf                     d   S r SSKJrJrJrJrJrJr  SSKrSSKJ	r	  SSK
Jr  SSKJr  SSKJr  SS	KJr  SS
KJrJrJrJrJrJr  SSKJrJr  SSKJr  SSKJr   SSK J!r!     S6S\S\\#\4   S\\$   S\$S\	RJ                  4
S jjr&\ " S S\	RN                  5      5       r(\ " S S\	RR                  5      5       r*S7S jr+\ " S S\R                  RJ                  5      5       r,S r-S r.S\/S\/S \/4S! jr0 " S" S#\	Rb                  5      r2 " S$ S%\	Rb                  5      r3 " S& S'\	Rh                  5      r5 " S( S)\	Rh                  5      r6 " S* S+\	Rh                  5      r7 " S, S-\	Rh                  5      r8 " S. S/\5      r9 " S0 S1\5      r: " S2 S3\5      r; " S4 S5\5      r<g! \" a
    SSKJ!r!   GNNf = f)8av  Normalization + Activation Layers

Provides Norm+Act fns for standard PyTorch norm layers such as
* BatchNorm
* GroupNorm
* LayerNorm

This allows swapping with alternative layers that are natively both norm + act such as
* EvoNorm (evo_norm.py)
* FilterResponseNorm (filter_response_norm.py)
* InplaceABN (inplace_abn.py)

Hacked together by / Copyright 2022 Ross Wightman
    )AnyDictListOptionalTypeUnionN)nn)
functional)FrozenBatchNorm2d   )register_notrace_module)create_act_layer)is_fast_normfast_group_normfast_layer_normfast_rms_norm
rms_norm2dfast_rms_norm2d)RmsNorm	RmsNorm2d)_assert)	LayerType)rms_norm	act_layer
act_kwargsinplace	apply_actreturnc                     U=(       d    0 nUR                  SU5        S nU(       a  [        U 40 UD6nUc  [        R                  " 5       $ U$ )Nr   )
setdefaultr   r	   Identity)r   r   r   r   acts        S/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/norm_act.py_create_actr$   *   sJ     !rJ)W-
Cy7J7K2;;=0S0    c                      ^  \ rS rSrSrSSSSS\R                  SSSSS4S\S\S	\S
\	S\	S\	S\
S\\\4   S\	S\\\R                         4U 4S jjjrS rSrU =r$ )BatchNormAct2d8   zBatchNorm + Activation

This module performs BatchNorm + Activation in a manner that will remain backwards
compatible with weights trained with separate bn, act. This is why we inherit from BN
instead of composing it as a .bn member.
h㈵>g?TNnum_featuresepsmomentumaffinetrack_running_statsr   r   r   r   
drop_layerc           	         >  XS.n[         TU ]  " U4UUUUS.UD6  U
b  U
" 5       O[        R                  " 5       U l        [        XxXS9U l        g ! [         a    [         TU ]  UUUUUS9   NQf = f)Ndevicedtype)r+   r,   r-   r.   r   r   r   )super__init__	TypeErrorr	   r!   dropr$   r"   )selfr*   r+   r,   r-   r.   r   r   r   r   r/   r2   r3   factory_kwargs	__class__s                 r#   r6   BatchNormAct2d.__init__@   s    	(.?NG!$7 !" %/$:JL	yf  	G!$7  	s   A A/.A/c           
      h   [        UR                  S:H  SUR                   S35        U R                  c  SnOU R                  nU R                  (       ak  U R                  (       aZ  U R
                  bM  U R
                  R                  S5        U R                  c  S[        U R
                  5      -  nOU R                  n U R                  (       a  SnO#U R                  S L =(       a    U R                  S L n [        R                  " UU R                  (       a  U R                  (       a  U R                  OS U R                  (       a  U R                  (       a  U R                  OS U R                  U R                  UUU R                  5      nU R                  U5      nU R!                  U5      nU$ )N   zexpected 4D input (got zD input)g        r   g      ?T)r   ndimr,   trainingr.   num_batches_trackedadd_floatrunning_meanrunning_varF
batch_normweightbiasr+   r8   r"   )r9   xexponential_average_factorbn_trainings       r#   forwardBatchNormAct2d.forwarde   sR   !6qvvhhGH
 == ),&)-&==T55''3((--a0==(14uT=U=U7V1V.15.	 ==K,,4T4;K;Kt;SK	
 LL%)]]d6N6NDTX$(MMT5M5MDSWKKII&HH

 IIaLHHQKr%   r"   r8   )__name__
__module____qualname____firstlineno____doc__r	   ReLUintrC   boolr   r   strr   r   r   Moduler6   rM   __static_attributes____classcell__r;   s   @r#   r'   r'   8   s     !(,"#%77)- 48#g#g #g 	#g
 #g "&#g #g !#g S#X#g #g !bii1#g #gJ0 0r%   r'   c                   \   ^  \ rS rSrS\R
                  S\R
                  4U 4S jjrSrU =r$ )SyncBatchNormAct   rJ   r   c                    > [         TU ]  U5      n[        U S5      (       a  U R                  U5      n[        U S5      (       a  U R	                  U5      nU$ )Nr8   r"   )r5   rM   hasattrr8   r"   )r9   rJ   r;   s     r#   rM   SyncBatchNormAct.forward   sJ    GOA4  		!A4Ar%    )	rP   rQ   rR   rS   torchTensorrM   rZ   r[   r\   s   @r#   r^   r^      s#     %,,  r%   r^   c           	         U n[        U [        R                  R                  R                  R
                  5      (       Ga  [        U [        5      (       ac  [        U R                  U R                  U R                  U R                  U R                  US9nU R                  Ul        U R                  Ul        OV[        R                  R                  U R                  U R                  U R                  U R                  U R                  U5      nU R                  (       a@  [        R                   " 5          U R"                  Ul        U R$                  Ul        S S S 5        U R&                  Ul        U R(                  Ul        U R*                  Ul        U R,                  Ul        [/        U S5      (       a  U R0                  Ul        U R3                  5        H   u  p4UR5                  U[7        XA5      5        M"     A U$ ! , (       d  f       N= f)N)process_groupqconfig)
isinstancerd   r	   modules	batchnorm
_BatchNormr'   r^   r*   r+   r,   r-   r.   r"   r8   SyncBatchNormno_gradrH   rI   rD   rE   rA   r@   ra   rh   named_children
add_moduleconvert_sync_batchnorm)modulerg   module_outputnamechilds        r#   rq   rq      s   M&%((**44??@@fn--,##

**+M !'

M!'M "HH22##

**M =='-}}$%+[[" ! &,%8%8"$*$6$6!,2,F,F)!'69%%$*NNM!,,.  '=e'ST / !s   5#G==
Hc                   $  ^  \ rS rSrSrSS\R                  SSSSS4S\S\S\	S	\
S
\\\4   S\	S\\\R                         4U 4S jjjrS\S\S\S\	S\\   S\\   S\\   4U 4S jjrS\R,                  S\R,                  4S jrS\4S jrSrU =r$ )FrozenBatchNormAct2d   a  
BatchNormAct2d where the batch statistics and the affine parameters are fixed

Args:
    num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)``
    eps (float): a value added to the denominator for numerical stability. Default: 1e-5
r)   TNr*   r+   r   r   r   r   r/   c
                   > XS.n
[         TU ]  5         X l        U R                  S[        R
                  " U40 U
D65        U R                  S[        R                  " U40 U
D65        U R                  S[        R                  " U40 U
D65        U R                  S[        R
                  " U40 U
D65        Ub  U" 5       O[        R                  " 5       U l	        [        XEXcS9U l        g )Nr1   rH   rI   rD   rE   r4   )r5   r6   r+   register_bufferrd   oneszerosr	   r!   r8   r$   r"   )r9   r*   r+   r   r   r   r   r/   r2   r3   ddr;   s              r#   r6   FrozenBatchNormAct2d.__init__   s     /Xuzz,'E"'EFVU[[%D%DE^U[[-L-LM]EJJ|,Jr,JK$.$:JL	yfr%   
state_dictprefixlocal_metadatastrictmissing_keysunexpected_keys
error_msgsc           	      B   > US-   nX;   a  X	 [         T	U ]  XX4XVU5        g )NrA   )r5   _load_from_state_dict)
r9   r   r   r   r   r   r   r   num_batches_tracked_keyr;   s
            r#   r   *FrozenBatchNormAct2d._load_from_state_dict   s4     #)+@"@"03%Wa	
r%   rJ   r   c                    U R                   R                  SSSS5      nU R                  R                  SSSS5      nU R                  R                  SSSS5      nU R                  R                  SSSS5      nX$U R
                  -   R                  5       -  nX5U-  -
  nX-  U-   nU R                  U R                  U5      5      nU$ )Nr   )	rH   reshaperI   rE   rD   r+   rsqrtr"   r8   )r9   rJ   wbrvrmscalerI   s           r#   rM   FrozenBatchNormAct2d.forward  s     KK2q!,IIaQ*%%aQ2&&q"a3$((]))++:~IHHTYYq\"r%   c                     U R                   R                   SU R                  R                  S    SU R                   SU R
                   S3$ )N(r   z, eps=z, act=))r;   rP   rH   shaper+   r"   )r9   s    r#   __repr__FrozenBatchNormAct2d.__repr__  sJ    ..))*!DKK,=,=a,@+AzQWX\X`X`Waabccr%   )r"   r8   r+   )rP   rQ   rR   rS   rT   r	   rU   rV   rC   rW   r   r   rX   r   r   r   rY   r6   dictr   r   rd   re   rM   r   rZ   r[   r\   s   @r#   rw   rw      s    "#%77)- 48gg g 	g
 !g S#Xg g !bii1g g.

 
 	

 
 3i
 c
 I
$ %,, d# d dr%   rw   c                 p   U n[        U [        [        45      (       GaI  [        U R                  5      nU R                  Ul        U R
                  Ul        U R
                  (       a  U R                  R                  R                  5       R                  5       UR                  l        U R                  R                  R                  5       R                  5       UR                  l        U R                  R                  UR                  l        U R                  R                  UR                  l        U R                  Ul        U R                  Ul        U R                  Ul        U$ [        U [         R"                  R$                  R&                  R(                  [         R"                  R$                  R&                  R*                  45      (       Ga'  [-        U R                  5      nU R                  Ul        U R
                  Ul        U R
                  (       a  U R                  R                  R                  5       R                  5       UR                  l        U R                  R                  R                  5       R                  5       UR                  l        U R                  R                  UR                  l        U R                  R                  UR                  l        U R                  Ul        U$ U R/                  5        H'  u  p#[1        U5      nXCLd  M  UR3                  X$5        M)     U$ )a  
Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers
of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively.

Args:
    module (torch.nn.Module): Any PyTorch module.

Returns:
    torch.nn.Module: Resulting module

Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
)ri   r'   r^   rw   r*   r-   rH   dataclonedetachrI   rD   rE   r+   r8   r"   rd   r	   rj   rk   BatchNorm2drm   r   ro   freeze_batch_norm_2drp   rr   resrt   ru   	new_childs        r#   r   r     s0    C&>+;<=="6#6#67!..]]
==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166**;;**  J 
FUXX--77CCUXXEUEUE_E_EmEmn	o	o 3 34!..]]
==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166** J	 "002KD,U3I%t/ 3 Jr%   c                 X   U n[        U [        5      (       Ga'  [        U R                  5      nU R                  (       a  U R
                  R                  R                  5       R                  5       UR
                  l        U R                  R                  R                  5       R                  5       UR                  l        U R                  R                  UR                  l        U R                  R                  UR                  l        U R                  Ul        U R                  Ul        U R                  Ul        U$ [        U [        5      (       Ga  [         R"                  R%                  U R                  5      nU R                  (       a  U R
                  R                  R                  5       R                  5       UR
                  l        U R                  R                  R                  5       R                  5       UR                  l        U R                  R                  UR                  l        U R                  R                  UR                  l        U R                  Ul        U$ U R'                  5        H'  u  p#[)        U5      nXCLd  M  UR+                  X$5        M)     U$ )a  
Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance
of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked
recursively and submodules are converted in place.

Args:
    module (torch.nn.Module): Any PyTorch module.

Returns:
    torch.nn.Module: Resulting module

Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
)ri   rw   r'   r*   r-   rH   r   r   r   rI   rD   rE   r+   r8   r"   r   rd   r	   r   ro   unfreeze_batch_norm_2drp   r   s        r#   r   r   B  s    C&.//V001==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166**;;** J 
F-	.	.hh""6#6#67==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166** J	 "002KD.u5I%t/ 3 Jr%   num_channels
num_groups
group_sizec                 0    U(       a  X-  S:X  d   eX-  $ U$ )Nr   rc   )r   r   r   s      r#   _num_groupsr   k  s%    (A---))r%   c                      ^  \ rS rSr% \R
                  R                  \   \S'   SSSSS\	R                  SSSSS4S\S\S	\S
\S\\   S\S\S\\\4   S\S\\\	R&                        4U 4S jjjrS rSrU =r$ )GroupNormActir  
_fast_norm    r)   TNr   r   r+   r-   r   r   r   r   r   r/   c           	         > [         TU ]  [        XU5      UUUUUS9  U
b  U
" 5       O[        R                  " 5       U l        [        XxXS9U l        [        5       U l	        g )Nr+   r-   r2   r3   r4   )
r5   r6   r   r	   r!   r8   r$   r"   r   r   )r9   r   r   r+   r-   r   r   r   r   r   r/   r2   r3   r;   s                r#   r6   GroupNormAct.__init__v  s_     	*= 	 	
 %/$:JL	yf&.r%   c                 \   U R                   (       a7  [        XR                  U R                  U R                  U R
                  5      nOA[        R                  " XR                  U R                  U R                  U R
                  5      nU R                  U5      nU R                  U5      nU$ N
r   r   r   rH   rI   r+   rF   
group_normr8   r"   r9   rJ   s     r#   rM   GroupNormAct.forward  l    ????DKKDHHUAQdiiRAIIaLHHQKr%   r   r"   r8   )rP   rQ   rR   rS   rd   jitFinalrW   __annotations__r	   rU   rV   rC   r   r   r   rX   r   r   rY   r6   rM   rZ   r[   r\   s   @r#   r   r   r  s    		%% !(,"#%77)- 48)) ) 	)
 ) !) ) !) S#X) ) !bii1) )8 r%   r   c                      ^  \ rS rSr% \R
                  R                  \   \S'   SSS\	R                  SSSSS4	S\S\S\S	\S
\S\\\4   S\S\\\	R&                        4U 4S jjjrS rSrU =r$ )GroupNorm1Acti  r   r)   TNr   r+   r-   r   r   r   r   r/   c           	         > [         TU ]  SXX9U
S9  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        [        5       U l        g )Nr   r   r4   	r5   r6   r	   r!   r8   r$   r"   r   r   )r9   r   r+   r-   r   r   r   r   r/   r2   r3   r;   s              r#   r6   GroupNorm1Act.__init__  sI     	L&W\]$.$:JL	yf&.r%   c                 \   U R                   (       a7  [        XR                  U R                  U R                  U R
                  5      nOA[        R                  " XR                  U R                  U R                  U R
                  5      nU R                  U5      nU R                  U5      nU$ r   r   r   s     r#   rM   GroupNorm1Act.forward  r   r%   r   rP   rQ   rR   rS   rd   r   r   rW   r   r	   rU   rV   rC   r   r   rX   r   r   r   rY   r6   rM   rZ   r[   r\   s   @r#   r   r     s    		%%
 "#%77)- 48)) ) 	)
 ) !) S#X) ) !bii1) )& r%   r   c                     ^  \ rS rSr% \R
                  R                  \   \S'   SSS\	R                  SSS4S\\\\   \R                  4   S\S\S	\S
\S\\\4   S\S\\\	R,                        4U 4S jjjrS rSrU =r$ )LayerNormActi  r   r)   TNnormalization_shaper+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " U4X#S.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        [        5       U l        g N)r+   elementwise_affiner4   r   r9   r   r+   r-   r   r   r   r   r/   kwargsr;   s             r#   r6   LayerNormAct.__init__  sK     	,[#[TZ[$.$:JL	yf&.r%   c                 \   U R                   (       a7  [        XR                  U R                  U R                  U R
                  5      nOA[        R                  " XR                  U R                  U R                  U R
                  5      nU R                  U5      nU R                  U5      nU$ r   )
r   r   normalized_shaperH   rI   r+   rF   
layer_normr8   r"   r   s     r#   rM   LayerNormAct.forward  ss    ??#8#8$++tyyRVRZRZ[AQ 5 5t{{DIItxxXAIIaLHHQKr%   r   )rP   rQ   rR   rS   rd   r   r   rW   r   r	   rU   r   rV   r   SizerC   r   r   rX   r   r   r   rY   r6   rM   rZ   r[   r\   s   @r#   r   r     s    		%%
 "#%77)- 48)!&sDIuzz'A!B) ) 	)
 ) !) S#X) ) !bii1) )$ r%   r   c                      ^  \ rS rSrSSS\R
                  SSS4S\\\\   \	R                  4   S\S\S\S	\S
\\\4   S\S\\\R&                        4U 4S jjjrS rSrU =r$ )LayerNormActFp32i  r)   TNr   r+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " U4X#S.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        g r   r5   r6   r	   r!   r8   r$   r"   r   s             r#   r6   LayerNormActFp32.__init__  sB     	,[#[TZ[$.$:JL	yfr%   c                    U R                   b  U R                   R                  5       OS nU R                  b  U R                  R                  5       OS n[        R                  " UR                  5       U R
                  X#U R                  5      R                  UR                  5      nU R                  U5      nU R                  U5      nU$ r   )rH   rC   rI   rF   r   r   r+   tor3   r8   r"   r9   rJ   rH   rI   s       r#   rM   LayerNormActFp32.forward  s    (,(?""$T$(II$9tyy tLLD$9$96RUUVWV]V]^IIaLHHQKr%   rO   )rP   rQ   rR   rS   r	   rU   r   rV   r   rd   r   rC   rW   r   r   rX   r   r   r   rY   r6   rM   rZ   r[   r\   s   @r#   r   r     s    
 "#%77)- 48g!&sDIuzz'A!Bg g 	g
 g !g S#Xg g !bii1g g  r%   r   c                      ^  \ rS rSr% \R
                  R                  \   \S'   SSS\	R                  SSS4S\S\S\S	\S
\S\\\4   S\S\\\	R&                        4U 4S jjjrS rSrU =r$ )LayerNormAct2di  r   r)   TNr   r+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " U4X#S.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        [        5       U l        g r   r   r9   r   r+   r-   r   r   r   r   r/   r   r;   s             r#   r6   LayerNormAct2d.__init__  sI     	T3TVT$.$:JL	yf&.r%   c                    UR                  SSSS5      nU R                  (       a7  [        XR                  U R                  U R
                  U R                  5      nOA[        R                  " XR                  U R                  U R
                  U R                  5      nUR                  SSSS5      nU R                  U5      nU R                  U5      nU$ Nr         r   )permuter   r   r   rH   rI   r+   rF   r   r8   r"   r   s     r#   rM   LayerNormAct2d.forward
  s    IIaAq!??#8#8$++tyyRVRZRZ[AQ 5 5t{{DIItxxXAIIaAq!IIaLHHQKr%   r   r   r\   s   @r#   r   r     s    		%%
 "#%77)- 48)) ) 	)
 ) !) S#X) ) !bii1) )"	 	r%   r   c                      ^  \ rS rSrSSS\R
                  SSS4S\S\S\S\S	\	S
\
\\4   S\S\\\R                        4U 4S jjjrS rSrU =r$ )LayerNormAct2dFp32i  r)   TNr   r+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " U4X#S.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        g r   r   r   s             r#   r6   LayerNormAct2dFp32.__init__  s@     	T3TVT$.$:JL	yfr%   c                    UR                  SSSS5      nU R                  b  U R                  R                  5       OS nU R                  b  U R                  R                  5       OS n[        R
                  " UR                  5       U R                  X#U R                  5      R                  UR                  5      nUR                  SSSS5      nU R                  U5      nU R                  U5      nU$ r   )r   rH   rC   rI   rF   r   r   r+   r   r3   r8   r"   r   s       r#   rM   LayerNormAct2dFp32.forward(  s    IIaAq!(,(?""$T$(II$9tyy tLLD$9$96RUUVWV]V]^IIaAq!IIaLHHQKr%   rO   )rP   rQ   rR   rS   r	   rU   rV   rC   rW   r   r   rX   r   r   r   rY   r6   rM   rZ   r[   r\   s   @r#   r   r     s    
 "#%77)- 48gg g 	g
 g !g S#Xg g !bii1g g  r%   r   c                      ^  \ rS rSrSrSSS\R                  SSS4S\S\S\	S	\	S
\
S\\\4   S\	S\\\R                         4U 4S jjjrS\R&                  S\R&                  4S jrSrU =r$ )
RmsNormActi3    RMSNorm + Activation for '2D' NCHW tensors

NOTE: It's currently (2025-05-10) faster to use an eager 2d kernel that does reduction
on dim=1 than to permute and use internal PyTorch F.rms_norm, this may change if something
like https://github.com/pytorch/pytorch/pull/150576 lands.
ư>TNr   r+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " SXUS.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        [        5       U l        g N)channelsr+   r-   r4   rc   r   r   s             r#   r6   RmsNormAct.__init__:  I     	Q,Q&Q$.$:JL	yf&.r%   rJ   r   c                    U R                   (       a,  [        XR                  U R                  U R                  5      nO+[        XR                  U R                  U R                  5      nU R                  U5      nU R                  U5      nU$ r   )r   r   r   rH   r+   r   r8   r"   r   s     r#   rM   RmsNormAct.forwardK  s`    ??a!6!6TXXNA114;;IAIIaLHHQKr%   r   rP   rQ   rR   rS   rT   r	   rU   rV   rC   rW   r   r   rX   r   r   r   rY   r6   rd   re   rM   rZ   r[   r\   s   @r#   r   r   3       "#%77)- 48)) ) 	)
 ) !) S#X) ) !bii1) )" %,,  r%   r   c                      ^  \ rS rSrSrSSS\R                  SSS4S\S\S\	S	\	S
\
S\\\4   S\	S\\\R                         4U 4S jjjrS\R&                  S\R&                  4S jrSrU =r$ )RmsNormActFp32iU  r   r   TNr   r+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " SXUS.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        g r   r   r   s             r#   r6   RmsNormActFp32.__init__\  @     	Q,Q&Q$.$:JL	yfr%   rJ   r   c                 ,   U R                   b  U R                   R                  5       OS n[        UR                  5       U R                  X R                  5      R                  UR                  5      nU R                  U5      nU R                  U5      nU$ r   )	rH   rC   r   r   r+   r   r3   r8   r"   r9   rJ   rH   s      r#   rM   RmsNormActFp32.forwardl  sk    (,(?""$TQWWY 5 5vxxHKKAGGTIIaLHHQKr%   rO   r   r\   s   @r#   r   r   U       "#%77)- 48gg g 	g
 g !g S#Xg g !bii1g g  %,,  r%   r   c                      ^  \ rS rSrSrSSS\R                  SSS4S\S\S\	S	\	S
\
S\\\4   S\	S\\\R                         4U 4S jjjrS\R&                  S\R&                  4S jrSrU =r$ )RmsNormAct2dit  r   r   TNr   r+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " SXUS.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        [        5       U l        g r   r   r   s             r#   r6   RmsNormAct2d.__init__{  r   r%   rJ   r   c                    U R                   (       a,  [        XR                  U R                  U R                  5      nO+[        XR                  U R                  U R                  5      nU R                  U5      nU R                  U5      nU$ r   )r   r   r   rH   r+   r   r8   r"   r   s     r#   rM   RmsNormAct2d.forward  s`    ??#8#8$++txxPA133T[[$((KAIIaLHHQKr%   r   r   r\   s   @r#   r  r  t  r   r%   r  c                      ^  \ rS rSrSrSSS\R                  SSS4S\S\S\	S	\	S
\
S\\\4   S\	S\\\R                         4U 4S jjjrS\R&                  S\R&                  4S jrSrU =r$ )RmsNormAct2dFp32i  r   r   TNr   r+   r-   r   r   r   r   r/   c	                    > [         T
U ]  " SXUS.U	D6  Ub  U" 5       O[        R                  " 5       U l        [        XVXtS9U l        g r   r   r   s             r#   r6   RmsNormAct2dFp32.__init__  r   r%   rJ   r   c                 ,   U R                   b  U R                   R                  5       OS n[        UR                  5       U R                  X R                  5      R                  UR                  5      nU R                  U5      nU R                  U5      nU$ r   )	rH   rC   r   r   r+   r   r3   r8   r"   r   s      r#   rM   RmsNormAct2dFp32.forward  sk    (,(?""$Tqwwy$"7"7JMMaggVIIaLHHQKr%   rO   r   r\   s   @r#   r
  r
    r  r%   r
  )NFTr   )=rT   typingr   r   r   r   r   r   rd   r	   torch.nnr
   rF   torchvision.ops.miscr   _fxr   
create_actr   	fast_normr   r   r   r   r   r   normr   r   trace_utilsr   r   torch.nn.functionalr   ImportErrorrX   rW   rY   r$   r   r'   rm   r^   rq   rw   r   r   rV   r   	GroupNormr   r   	LayerNormr   r   r   r   r   r   r  r
  rc   r%   r#   <module>r     s   : 9   $ 2 ( (  %   $, &*"'	11cN1 $1 	1
 YY1 \R^^ \ \~ r''  (V @d588?? @d @dF)X&Rc s  '2<< 'TBLL @2<< >r|| 6R\\ @ : DW >9 Dy a  $##$s   F F/.F/