
    RЦi?              	       F   S r SSKJrJrJrJr  SSKrSSKJr  SSK	Js  J
r  SSKJr  SSKJr  S%S\4S jjrS%S\4S	 jjrS%S\4S
 jjrS&S\\\\   4   S\4S jjrS'S\S\S\4S jjrS(S\S\S\S\4S jjrS)S\S\4S jjr " S S\R4                  5      r " S S\R4                  5      r " S S\R4                  5      r " S S\R4                  5      r " S S\5      r " S S\R4                  5      r  " S S \ 5      r! " S! S"\R4                  5      r" " S# S$\"5      r#g)*a6  EvoNorm in PyTorch

Based on `Evolving Normalization-Activation Layers` - https://arxiv.org/abs/2004.02967
@inproceedings{NEURIPS2020,
 author = {Liu, Hanxiao and Brock, Andy and Simonyan, Karen and Le, Quoc},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {13539--13550},
 publisher = {Curran Associates, Inc.},
 title = {Evolving Normalization-Activation Layers},
 url = {https://proceedings.neurips.cc/paper/2020/file/9d4c03631b8b0c85ae08bf05eda37d0f-Paper.pdf},
 volume = {33},
 year = {2020}
}

An attempt at getting decent performing EvoNorms running in PyTorch.
While faster than other PyTorch impl, still quite a ways off the built-in BatchNorm
in terms of memory usage and throughput on GPUs.

I'm testing these modules on TPU w/ PyTorch XLA. Promising start but
currently working around some issues with builtin torch/tensor.var/std. Unlike
GPU, similar train speeds for EvoNormS variants and BatchNorm.

Hacked together by / Copyright 2020 Ross Wightman
    )OptionalSequenceTypeUnionN   )create_act_layer)_assertepsc                     U R                  5       R                  SSSS9R                  U5      R                  5       R	                  U R
                  5      nUR                  U R                  5      $ )N      FTdimunbiasedkeepdim)floatvaraddsqrttodtypeexpandshapexr
   stds      S/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/evo_norm.pyinstance_stdr   $   sU    
'')--FUD-
A
E
Ec
J
O
O
Q
T
TUVU\U\
]C::agg    c                     [        U SS9R                  U5      R                  5       nUR                  U R                  5      $ )Nr   )r   )
manual_varr   r   r   r   r   s      r   instance_std_tpur#   )   s5    
QF
#
'
'
,
1
1
3C::aggr    c                     U R                  5       R                  5       R                  SSS9R                  U5      R	                  5       R                  U R                  5      nUR                  U R                  5      $ )Nr   Tr   r   )	r   squaremeanr   r   r   r   r   r   )r   r
   rmss      r   instance_rmsr)   /   s\    
'')



!
!fd
!
;
?
?
D
I
I
K
N
Nqww
WC::aggr    r   diff_sqmc                     U R                  USS9nU(       a(  X -  R                  USS9X3-  -
  R                  S5      nU$ X-
  X-
  -  R                  USS9nU$ )NTr%   r   )r'   clamp)r   r   r*   xmr   s        r   r"   r"   4   si    	
C	&B||T|2bg>EEaH J 16"((S$(?Jr    groupsflattenc                 :   U R                   u  pEpgU R                  n[        XQ-  S:H  S5        U(       a^  U R                  XAS5      n U R	                  5       R                  SSSS9R                  U5      R                  5       R                  U5      n	O`U R                  XAXQ-  Xg5      n U R	                  5       R                  SSSS9R                  U5      R                  5       R                  U5      n	U	R                  U R                   5      R                  XEXg5      $ )	Nr    r   FTr   r   r      )
r   r   r	   reshaper   r   r   r   r   r   )
r   r.   r
   r/   BCHWx_dtyper   s
             r   	group_stdr;   >   s    JA!ggGAJ!OR IIa$ggimmE4m@DDSINNPSST[\IIaa3ggimm	E4mHLLSQVVX[[\cd::agg&&qQ22r    c                 \   U R                   u  pVpx[        Xa-  S:H  S5        U(       a  U R                  XQS5      n [        U SUS9n	O U R                  XQXa-  Xx5      n [        U SUS9n	U	R	                  U5      R                  5       R                  U R                   5      R                  XVXx5      $ )Nr   r1   r2   )r   r*   r3   )r   r	   r5   r"   r   r   r   )
r   r.   r
   r*   r/   r6   r7   r8   r9   r   s
             r   group_std_tpur=   K   s     JA!AJ!OR IIa$X6IIaa3	H=773<%%agg.66qQBBr    c                    U R                   u  p4pV[        XA-  S:H  S5        U R                  nU R                  X1XA-  XV5      n U R	                  5       R                  5       R                  SSS9R                  U5      R                  5       R                  U5      nUR                  U R                   5      R                  X4XV5      $ )Nr   r1   r3   Tr%   )r   r	   r   r5   r   r&   r'   r   sqrt_r   r   )	r   r.   r
   r6   r7   r8   r9   r:   r(   s	            r   	group_rmsr@   Z   s    JA!AJ!OR ggG			!Q[!/A
'')



!
!i
!
>
B
B3
G
M
M
O
R
RSZ
[C::agg&&qQ22r    c            	       V   ^  \ rS rSr     S
S\S\S\S\4U 4S jjjrS rS r	S	r
U =r$ )EvoNorm2dB0c   num_features	apply_actmomentumr
   c                   > XVS.n[         T	U ]  5         X l        X0l        X@l        [
        R                  " [        R                  " U40 UD65      U l	        [
        R                  " [        R                  " U40 UD65      U l
        U(       a+  [
        R                  " [        R                  " U40 UD65      OS U l        U R                  S[        R                  " U40 UD65        U R                  5         g Ndevicer   running_var)super__init__rE   rF   r
   nn	Parametertorchemptyweightbiasvregister_bufferonesreset_parameters
selfrD   rE   rF   r
   rJ   r   _dd	__class__s
            r   rM   EvoNorm2dB0.__init__d   s     /" ll5;;|#Br#BCLL\!@R!@A	BKekk,="=>QU]EJJ|,Jr,JKr    c                    [         R                  R                  U R                  5        [         R                  R	                  U R
                  5        U R                  b*  [         R                  R                  U R                  5        g g NrN   initones_rR   zeros_rS   rT   rY   s    r   rW   EvoNorm2dB0.reset_parametersz   M    
dkk"
tyy!66GGMM$&&! r    c                    [        UR                  5       S:H  S5        UR                  nSnU R                  Gbd  U R                  (       a  UR                  5       R                  SSS9nUR                  5       UR                  S   -  nU R                  R                  U R                  SU R                  -
  -  UR                  5       U R                  -  XUS-
  -  -  -   5        OU R                  nUR                  U R                  5      R                  5       R!                  U5      R#                  U5      R%                  U5      nU R                  R!                  U5      R#                  U5      nX-  ['        XR                  5      -   nXR)                  U5      -  nXR*                  R!                  U5      R#                  U5      -  U R,                  R!                  U5      R#                  U5      -   $ Nr4   expected 4D inputr   r2   r   r   )r   r   r   F)r   r   r   )r	   r   r   rT   trainingr   r   numelr   rK   copy_rF   detachr   r
   r?   r   view	expand_asr   maxrR   rS   )	rY   r   r:   v_shaper   nleftrT   rights	            r   forwardEvoNorm2dB0.forward   s   112''66}}ggimm	EmBGGI
*  &&$$DMM(9:JJL4==0AQK@AB &&77488$**,//8==gFPPQRSD		'"''0AELHH55EHHUO#A;;>>'*//88499<<;P;U;UV];^^^r    )rE   rS   r
   rF   rT   rR   )T皙?MbP?NN__name__
__module____qualname____firstlineno__intboolr   rM   rW   rv   __static_attributes____classcell__r\   s   @r   rB   rB   c   sU     #!     	 
    ,"_ _r    rB   c            	       V   ^  \ rS rSr     S
S\S\S\S\4U 4S jjjrS rS r	S	r
U =r$ )EvoNorm2dB1   rD   rE   rF   r
   c                 |  > XVS.n[         T	U ]  5         X l        X0l        X@l        [
        R                  " [        R                  " U40 UD65      U l	        [
        R                  " [        R                  " U40 UD65      U l
        U R                  S[        R                  " U40 UD65        U R                  5         g rH   rL   rM   rE   rF   r
   rN   rO   rP   rQ   rR   rS   rU   rV   rW   rX   s
            r   rM   EvoNorm2dB1.__init__        /" ll5;;|#Br#BCLL\!@R!@A	]EJJ|,Jr,JKr    c                     [         R                  R                  U R                  5        [         R                  R	                  U R
                  5        g r_   rN   ra   rb   rR   rc   rS   rd   s    r   rW   EvoNorm2dB1.reset_parameters   *    
dkk"
tyy!r    c                    [        UR                  5       S:H  S5        UR                  nSnU R                  (       GaQ  U R                  (       a  UR                  5       R                  SSS9nUR                  5       UR                  S   -  nU R                  R                  U R                  SU R                  -
  -  UR                  5       R                  U R                  R                  5      U R                  -  XUS-
  -  -  -   5        OU R                  nUR                  U5      R                  U5      nUR                  U R                   5      R#                  5       nUS-   [%        XR                   5      -  nXR'                  U5      -  nXR(                  R                  U5      R                  U5      -  U R*                  R                  U5      R                  U5      -   $ rh   r	   r   r   rE   rk   r   r   rl   r   rK   rm   rF   rn   r   ro   r   r
   r?   r)   rq   rR   rS   rY   r   r:   rr   r   rs   rt   ru   s           r   rv   EvoNorm2dB1.forward   ss   112''>>>}}ggimm	EmBGGI
*  &&$$DMM(9:JJLOOD$4$4$:$:;dmmKqXYTY{[\] &&&&/&&w/C77488$**,DUl1hh77EHHUO#A;;##G,//88499>>';R;U;UV];^^^r    rE   rS   r
   rF   rR   Trx   h㈵>NNrz   r   s   @r   r   r      U     #!     	 
    *"_ _r    r   c            	       V   ^  \ rS rSr     S
S\S\S\S\4U 4S jjjrS rS r	S	r
U =r$ )EvoNorm2dB2   rD   rE   rF   r
   c                 |  > XVS.n[         T	U ]  5         X l        X0l        X@l        [
        R                  " [        R                  " U40 UD65      U l	        [
        R                  " [        R                  " U40 UD65      U l
        U R                  S[        R                  " U40 UD65        U R                  5         g rH   r   rX   s
            r   rM   EvoNorm2dB2.__init__   r   r    c                     [         R                  R                  U R                  5        [         R                  R	                  U R
                  5        g r_   r   rd   s    r   rW   EvoNorm2dB2.reset_parameters   r   r    c                    [        UR                  5       S:H  S5        UR                  nSnU R                  (       GaN  U R                  (       a  UR                  5       R                  SSS9nUR                  5       UR                  S   -  nU R                  R                  U R                  SU R                  -
  -  UR                  5       R                  U R                  R                  5      U R                  -  XUS-
  -  -  -   5        OU R                  nUR                  U5      R                  U5      nUR                  U R                   5      R#                  5       n[%        XR                   5      U-
  nXR'                  U5      -  nXR(                  R                  U5      R                  U5      -  U R*                  R                  U5      R                  U5      -   $ rh   r   r   s           r   rv   EvoNorm2dB2.forward   so   112''>>>}}ggimm	EmBGGI
*  &&$$DMM(9:JJLOOD$4$4$:$:;dmmKqXYTY{[\] &&&&/&&w/C77488$**,D HH-1EHHUO#A;;##G,//88499>>';R;U;UV];^^^r    r   r   rz   r   s   @r   r   r      r   r    r   c                   b   ^  \ rS rSr      SS\S\S\\   S\S\4
U 4S jjjrS r	S	 r
S
rU =r$ )EvoNorm2dS0   rD   r.   
group_sizerE   r
   c                   > XgS.n	[         T
U ]  5         X@l        U(       a  X-  S:X  d   eX-  U l        OX l        XPl        [
        R                  " [        R                  " U40 U	D65      U l	        [
        R                  " [        R                  " U40 U	D65      U l
        U(       a+  [
        R                  " [        R                  " U40 U	D65      OS U l        U R                  5         g NrI   r   )rL   rM   rE   r.   r
   rN   rO   rP   rQ   rR   rS   rT   rW   )rY   rD   r.   r   rE   r
   rJ   r   rZ   r[   r\   s             r   rM   EvoNorm2dS0.__init__   s     /",111&4DK Kll5;;|#Br#BCLL\!@R!@A	BKekk,="=>QUr    c                    [         R                  R                  U R                  5        [         R                  R	                  U R
                  5        U R                  b*  [         R                  R                  U R                  5        g g r_   r`   rd   s    r   rW   EvoNorm2dS0.reset_parameters  rf   r    c                    [        UR                  5       S:H  S5        UR                  nSnU R                  b`  U R                  R	                  U5      R                  U5      nXU-  R                  5       -  [        XR                  U R                  5      -  nXR                  R	                  U5      R                  U5      -  U R                  R	                  U5      R                  U5      -   $ Nr4   ri   rj   )r	   r   r   rT   ro   r   sigmoidr;   r.   r
   rR   rS   )rY   r   r:   rr   rT   s        r   rv   EvoNorm2dS0.forward  s    112''66G$''0AUOO%%	![[$(((KKA;;##G,//88499>>';R;U;UV];^^^r    )rE   rS   r
   r.   rT   rR   )    NTr   NN)r{   r|   r}   r~   r   r   r   r   rM   rW   rv   r   r   r   s   @r   r   r      sf     (,"     !	 
      4"_ _r    r   c                   \   ^  \ rS rSr      S
S\S\S\\   S\S\4
U 4S jjjrS r	S	r
U =r$ )EvoNorm2dS0ai  rD   r.   r   rE   r
   c           
      ,   > [         T	U ]  UUUUUUUS9  g )N)r.   r   rE   r
   rJ   r   rL   rM   )
rY   rD   r.   r   rE   r
   rJ   r   rZ   r\   s
            r   rM   EvoNorm2dS0a.__init__  s,     	! 	 	
r    c                    [        UR                  5       S:H  S5        UR                  nSn[        XR                  U R
                  5      nU R                  b?  U R                  R                  U5      R                  U5      nXU-  R                  5       -  nX-  nXR                  R                  U5      R                  U5      -  U R                  R                  U5      R                  U5      -   $ r   )r	   r   r   r;   r.   r
   rT   ro   r   r   rR   rS   )rY   r   r:   rr   drT   s         r   rv   EvoNorm2dS0a.forward2  s    112''adhh/66G$''0AUOO%%AE;;##G,//88499>>';R;U;UV];^^^r     )r   NTry   NN)r{   r|   r}   r~   r   r   r   r   rM   rv   r   r   r   s   @r   r   r     sa     (,"

 
 !	

 
 
 
*	_ 	_r    r   c                      ^  \ rS rSr       SS\S\S\\   S\S\\\R                        S\
4U 4S jjjrS	 rS
 rSrU =r$ )EvoNorm2dS1i>  rD   r.   r   rE   	act_layerr
   c	                   > XxS.n
[         TU ]  5         U=(       d    [        R                  nX@l        Ub  U(       a  [        U5      U l        O[        R                  " 5       U l        U(       a  X-  S:X  d   eX-  U l        OX l        X`l	        SU l
        [        R                  " [        R                  " U40 U
D65      U l        [        R                  " [        R                  " U40 U
D65      U l        U R!                  5         g )NrI   r   F)rL   rM   rN   SiLUrE   r   actIdentityr.   r
   pre_act_normrO   rP   rQ   rR   rS   rW   rY   rD   r.   r   rE   r   r
   rJ   r   rZ   r[   r\   s              r   rM   EvoNorm2dS1.__init__?  s     /(	" Y'	2DH{{}DH,111&4DK K!ll5;;|#Br#BCLL\!@R!@A	r    c                     [         R                  R                  U R                  5        [         R                  R	                  U R
                  5        g r_   r   rd   s    r   rW   EvoNorm2dS1.reset_parameters_  r   r    c                    [        UR                  5       S:H  S5        UR                  nSnU R                  (       a2  U R	                  U5      [        XR                  U R                  5      -  nXR                  R                  U5      R                  U5      -  U R                  R                  U5      R                  U5      -   $ r   )r	   r   r   rE   r   r;   r.   r
   rR   ro   r   rS   rY   r   r:   rr   s       r   rv   EvoNorm2dS1.forwardc      112''>>i;;AAA;;##G,//88499>>';R;U;UV];^^^r    )r   rE   rS   r
   r.   r   rR   r   NTNr   NNr{   r|   r}   r~   r   r   r   r   rN   Moduler   rM   rW   rv   r   r   r   s   @r   r   r   >  s     (,"37     !	 
    RYY0     @"_ _r    r   c                      ^  \ rS rSr       SS\S\S\\   S\S\\\R                        S\
4U 4S jjjrS	 rS
rU =r$ )EvoNorm2dS1ail  rD   r.   r   rE   r   r
   c	                 .   > [         T
U ]  UUUUUUUUS9  g N)r.   r   rE   r   r
   rJ   r   r   rY   rD   r.   r   rE   r   r
   rJ   r   rZ   r\   s             r   rM   EvoNorm2dS1a.__init__m  /     	! 	 		
r    c                 j   [        UR                  5       S:H  S5        UR                  nSnU R                  U5      [	        XR
                  U R                  5      -  nXR                  R                  U5      R                  U5      -  U R                  R                  U5      R                  U5      -   $ r   )r	   r   r   r   r;   r.   r
   rR   ro   r   rS   r   s       r   rv   EvoNorm2dS1a.forward      112''HHQK)A{{DHH==;;##G,//88499>>';R;U;UV];^^^r    r   r   NTNry   NNr{   r|   r}   r~   r   r   r   r   rN   r   r   rM   rv   r   r   r   s   @r   r   r   l  {     (,"37

 
 !	

 
  RYY0
 
 
._ _r    r   c                      ^  \ rS rSr       SS\S\S\\   S\S\\\R                        S\
4U 4S jjjrS	 rS
 rSrU =r$ )EvoNorm2dS2i  rD   r.   r   rE   r   r
   c	                    > XxS.n
[         TU ]  5         U=(       d    [        R                  nX@l        Ub  U(       a  [        U5      U l        O[        R                  " 5       U l        U(       a  X-  S:X  d   eX-  U l        OX l        X`l	        [        R                  " [        R                  " U40 U
D65      U l        [        R                  " [        R                  " U40 U
D65      U l        U R                  5         g r   )rL   rM   rN   r   rE   r   r   r   r.   r
   rO   rP   rQ   rR   rS   rW   r   s              r   rM   EvoNorm2dS2.__init__  s     /(	" Y'	2DH{{}DH,111&4DK Kll5;;|#Br#BCLL\!@R!@A	r    c                     [         R                  R                  U R                  5        [         R                  R	                  U R
                  5        g r_   r   rd   s    r   rW   EvoNorm2dS2.reset_parameters  r   r    c                    [        UR                  5       S:H  S5        UR                  nSnU R                  (       a2  U R	                  U5      [        XR                  U R                  5      -  nXR                  R                  U5      R                  U5      -  U R                  R                  U5      R                  U5      -   $ r   )r	   r   r   rE   r   r@   r.   r
   rR   ro   r   rS   r   s       r   rv   EvoNorm2dS2.forward  r   r    )r   rE   rS   r
   r.   rR   r   r   r   s   @r   r   r     s     (,"37     !	 
    RYY0     >"_ _r    r   c                      ^  \ rS rSr       SS\S\S\\   S\S\\\R                        S\
4U 4S jjjrS	 rS
rU =r$ )EvoNorm2dS2ai  rD   r.   r   rE   r   r
   c	                 .   > [         T
U ]  UUUUUUUUS9  g r   r   r   s             r   rM   EvoNorm2dS2a.__init__  r   r    c                 j   [        UR                  5       S:H  S5        UR                  nSnU R                  U5      [	        XR
                  U R                  5      -  nXR                  R                  U5      R                  U5      -  U R                  R                  U5      R                  U5      -   $ r   )r	   r   r   r   r@   r.   r
   rR   ro   r   rS   r   s       r   rv   EvoNorm2dS2a.forward  r   r    r   r   r   r   s   @r   r   r     r   r    r   )r   )F)r   r   F)r   r   FF)r   r   )$__doc__typingr   r   r   r   rP   torch.nnrN   torch.nn.functional
functionalF
create_actr   trace_utilsr	   r   r   r#   r)   r   r   r"   r;   r=   r@   r   rB   r   r   r   r   r   r   r   r   r   r    r   <module>r      s[  2 3 2     (   
U  
uS(3-/0 D 
3 
3 
3t 
3CS CE CD C[_ C3 3 3/_")) /_d+_")) +_\+_")) +_\(_")) (_V_; _D+_")) +_\_; _@*_")) *_Z_; _r    