
    RЦi~              	          S r SSKrSSKJr  SSKJrJrJrJrJ	r	J
r
  SSKrSSKJr  SSKJrJr  SSKJrJrJrJrJrJrJrJrJrJrJrJr  SSKJr  SS	K J!r!  SS
K"J#r#  SSK$J%r%J&r&  S/r'SSSSS.r(SSSSS.r)SSSSS.r* " S S\RV                  5      r, " S S\R                  RV                  5      r- " S S\R                  RV                  5      r. " S  S!\R                  RV                  5      r/ " S" S#\RV                  5      r0 " S$ S%\RV                  5      r1 " S& S'\RV                  5      r2 " S( S)\Rf                  5      r4 " S* S+\RV                  5      r5 " S, S\RV                  5      r6S7S- jr7\%" \7" S.S/9\7" S.S/9\7" S.S/9\7" S.S/9S0.5      r8S8S1 jr9\&S8S2\64S3 jj5       r:\&S8S2\64S4 jj5       r;\&S8S2\64S5 jj5       r<\&S8S2\64S6 jj5       r=g)9aI  EfficientFormer-V2

@article{
    li2022rethinking,
    title={Rethinking Vision Transformers for MobileNet Size and Speed},
    author={Li, Yanyu and Hu, Ju and Wen, Yang and Evangelidis, Georgios and Salahi, Kamyar and Wang, Yanzhi and Tulyakov, Sergey and Ren, Jian},
    journal={arXiv preprint arXiv:2212.08059},
    year={2022}
}

Significantly refactored and cleaned up for timm from original at: https://github.com/snap-research/EfficientFormer

Original code licensed Apache 2.0, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2023, Ross Wightman
    N)partial)DictListOptionalTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)create_conv2dcreate_norm_layerget_act_layerget_norm_layerConvNormActLayerScale2dDropPathcalculate_drop_path_ratestrunc_normal_	to_2tuple	to_ntuplendgrid   )build_model_with_cfg)feature_take_indices)checkpoint_seq)generate_default_cfgsregister_modelEfficientFormerV2)(   P        )    @      i   )r$   0   x      )r$   r'   `      )LS2S1S0)   r0      
   )   r3         )   r6   	      )   r9   r8   r3   )r3   r3   )r3   r3   r3   r3   r6   r6   r6   r6   r6   r6   r6   r3   r3   r3   r3   )
r3   r3   r3   r6   r6   r6   r6   r3   r3   r3   )r3   r3   )r3   r3   r6   r6   r6   r6   r6   r6   r3   r3   r3   r3   )r3   r3   r6   r6   r6   r6   r3   r3   )r3   r3   )	r3   r3   r6   r6   r6   r6   r3   r3   r3   )r3   r3   r6   r6   r3   r3   )r3   r3   )r3   r6   r6   r6   r3   r3   )r3   r6   r6   r3   c                      ^  \ rS rSr          SS\S\S\S\S\\\4   S\S\S	\S
\S\\	   4U 4S jjjr
S rSrU =r$ )ConvNormE   in_channelsout_channelskernel_sizestridepaddingdilationgroupsbias
norm_layernorm_kwargsc           	         > XS.nU
=(       d    0 n
[         TU ]  5         [        UUU4UUUUUS.UD6U l        [	        X40 U
DUD6U l        g )Ndevicedtype)r@   rA   rB   rC   rD   )super__init__r   convr   bn)selfr=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rI   rJ   dd	__class__s                 ]/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/efficientformer_v2.pyrL   ConvNorm.__init__F   so     /!'R!

 

 

	 $JRRrR    c                 J    U R                  U5      nU R                  U5      nU$ N)rM   rN   rO   xs     rR   forwardConvNorm.forwarde   s!    IIaLGGAJrT   )rN   rM   )
r   r    r   r   Tbatchnorm2dNNN)__name__
__module____qualname____firstlineno__intr	   strboolr   r   rL   rY   __static_attributes____classcell__rQ   s   @rR   r;   r;   E   s    
  !')+*.SS S 	S
 S 38_S S S S S "$S S> rT   r;   c                   p  ^  \ rS rSr% \\\R                  4   \S'   SSSSS\	R                  SSS4	S	\S
\S\S\S\\\\\4   4   S\\	R                     S\\   4U 4S jjjr\R$                  " 5       SU 4S jj5       rSS jrSS jrSS jrSS jrS\R0                  S\R                  4S jrS rSrU =r$ )Attention2dk   attention_bias_cacher#   r$   r5   r3      Ndimkey_dim	num_heads
attn_ratio
resolution	act_layerr@   c
           	      ~  > XS.n
[         TU ]  5         X0l        US-  U l        X l        [        U5      nUba  [        U Vs/ s H  n[        R                  " X-  5      PM     sn5      n[        X4SXqS.U
D6U l
        [        R                  " USS9U l        OS U l
        S U l        XPl        U R                  S   U R                  S   -  U l        [!        XB-  5      U l        [!        XB-  5      U-  U l        X@l        U R                  U R                  -  n[        X40 U
D6U l        [        X40 U
D6U l        [        XR$                  40 U
D6U l        [        U R$                  U R$                  4SU R$                  S	.U
D6U l        [        R0                  " U R                  U R                  4S
S0U
D6U l        [        R0                  " U R                  U R                  4S
S0U
D6U l        U" 5       U l        [        U R$                  US40 U
D6U l        [:        R                  R=                  [:        R>                  " X0R                  40 U
D65      U l         U RC                  S[:        R>                  " U R                  U R                  4U[:        RD                  S9SS9  0 U l#        U RI                  5         g s  snf )NrH         r6   r?   r@   rC   bilinear)scale_factormoder   r   )r?   rC   r?   attention_bias_idxsF
persistent)%rK   rL   rn   scalerm   r   tuplemathceilr;   stride_convnnUpsampleupsamplerp   Nra   ddhro   qkvv_localConv2dtalking_head1talking_head2actprojtorch	Parameteremptyattention_biasesregister_bufferlongrj   reset_parameters)rO   rl   rm   rn   ro   rp   rq   r@   rI   rJ   rP   rkhrQ   s                rR   rL   Attention2d.__init__n   sJ    /"_
z*
zJz!		!* 5zJKJ'aaa^`aDKKV*MDM#D DM$#dooa&88Z)*j*+i7$\\DNN*#(R(#(R(#ww-"-VaVSUVYYt~~t~~[ST[XZ[YYt~~t~~[ST[XZ[;TWWc133	 % 2 25;;y&&3WTV3W X!KK(uzzJ 	 	

 %'! 	C  Ks   "J:c                 f   > [         TU ]  U5        U(       a  U R                  (       a  0 U l        g g g rV   rK   trainrj   rO   rw   rQ   s     rR   r   Attention2d.train   )    dD--(*D% .4rT   returnc                 v    [         R                  R                  U R                  5        U R	                  5         gz"Initialize parameters and buffers.Nr   initzeros_r   _init_buffersrO   s    rR   r   Attention2d.reset_parameters   $    
t,,-rT   c                    [         R                  " [        [         R                  " U R                  S   U[         R
                  S9[         R                  " U R                  S   U[         R
                  S95      5      R                  S5      nUSSS2S4   USSSS24   -
  R                  5       nUS   U R                  S   -  US   -   nU$ )5Compute relative position indices for attention bias.r   rH   r   .Nr   stackr   arangerp   r   flattenabs)rO   rI   posrel_poss       rR   _compute_attention_bias_idxs(Attention2d._compute_attention_bias_idxs   s    kk&LL+F%**MLL+F%**M
  71: 	 sAt|$s3a<'88==?1: 22gaj@rT   c                     U R                   R                  U R                  U R                   R                  S95        0 U l        gz.Compute and fill non-persistent buffer values.)rI   Nrx   copy_r   rI   rj   r   s    rR   r   Attention2d._init_buffers   =      &&--T5M5M5T5T-U	
 %'!rT   c                 $    U R                  5         gz"Initialize non-persistent buffers.Nr   r   s    rR   init_non_persistent_buffers'Attention2d.init_non_persistent_buffers       rT   rI   c                 J   [         R                  R                  5       (       d  U R                  (       a  U R                  S S 2U R
                  4   $ [        U5      nX R                  ;  a*  U R                  S S 2U R
                  4   U R                  U'   U R                  U   $ rV   r   jit
is_tracingtrainingr   rx   rb   rj   rO   rI   
device_keys      rR   get_attention_biases Attention2d.get_attention_biases       99!!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88rT   c                    UR                   u  p#pEU R                  b  U R                  U5      nU R                  U5      R                  X R                  SU R
                  5      R                  SSSS5      nU R                  U5      R                  X R                  SU R
                  5      R                  SSSS5      nU R                  U5      nU R                  U5      n	UR                  X R                  SU R
                  5      R                  SSSS5      nXg-  U R                  -  n
XR                  UR                  5      -   n
U R                  U
5      n
U
R                  SS9n
U R                  U
5      n
X-  R!                  SS5      nUR                  X R"                  U R$                  S   U R$                  S   5      U	-   nU R&                  b  U R'                  U5      nU R)                  U5      nU R+                  U5      nU$ Nr   r   r6   r9   rl   )shaper   r   reshapern   r   permuter   r   r   r{   r   rI   r   softmaxr   	transposer   rp   r   r   r   rO   rX   BCHWr   r   r   r   attns              rR   rY   Attention2d.forward   s   WW
a'  #AFF1IaTVV<DDQ1aPFF1IaTVV<DDQ1aPFF1I,,q/IIaTVV4<<Q1aH#//99!!$'|||#!!$'X  A&IIa$//!"4dooa6HIGS==$a AHHQKIIaLrT   )r   r   rj   r   ro   r   r   r   rm   rn   r   r   rp   r{   r   r   r   r   r   r   Tr   NrV   )r]   r^   r_   r`   r   rb   r   Tensor__annotations__r   GELUra   r	   r   r   Moduler   rL   no_gradr   r   r   r   r   rI   r   rY   rd   re   rf   s   @rR   rh   rh   k   s    sELL011 67)+$(5 5  5  	5 
 5  c5c?235  BII5  SM5  5 n ]]_+ +

'95<< 9ELL 9 rT   rh   c                   B   ^  \ rS rSr  SS\S\4U 4S jjjrS rSrU =r$ )LocalGlobalQuery   in_dimout_dimc                    > X4S.n[         TU ]  5         [        R                  " SSS5      U l        [        R
                  " X4SSSUS.UD6U l        [        XS40 UD6U l        g )NrH   r   r9   r   r6   )r?   r@   rA   rC   )	rK   rL   r   	AvgPool2dpoolr   localr;   r   )rO   r   r   rI   rJ   rP   rQ   s         rR   rL   LocalGlobalQuery.__init__   sb     /LLAq)	YYvg1QPQZ`gdfg
Va626	rT   c                 t    U R                  U5      nU R                  U5      nX#-   nU R                  U5      nU$ rV   r   r   r   )rO   rX   local_qpool_qr   s        rR   rY   LocalGlobalQuery.forward   s6    **Q-1IIaLrT   r   )NN)	r]   r^   r_   r`   ra   rL   rY   rd   re   rf   s   @rR   r   r      s1    
 77 7 7 rT   r   c                   p  ^  \ rS rSr% \\\R                  4   \S'   SSSSSS\	R                  SS4	S	\S
\S\S\S\\\\\4   4   S\\   S\\	R                      4U 4S jjjr\R$                  " 5       SU 4S jj5       rSS jrSS jrSS jrSS jrS\R0                  S\R                  4S jrS rSrU =r$ )Attention2dDownsample   rj   r#      r5   r3   rk   Nrl   rm   rn   ro   rp   r   rq   c
           	        > XS.n
[         TU ]  5         X0l        US-  U l        X l        [        U5      U l        [        U R                   Vs/ s H  n[        R                  " US-  5      PM     sn5      U l
        U R                  S   U R                  S   -  U l        U R                  S   U R                  S   -  U l        [        XB-  5      U l        [        XB-  5      U-  U l        X@l        U=(       d    UU l        U R                  U R                  -  n[%        X40 U
D6U l        [)        XS40 U
D6U l        [)        XR                  S40 U
D6U l        [)        U R                  U R                  4SSU R                  S.U
D6U l        U" 5       U l        [)        U R                  U R"                  S40 U
D6U l        [4        R6                  " [8        R:                  " X0R                  40 U
D65      U l        U R?                  S[8        R:                  " U R                  U R                  4U[8        R@                  S9S	S
9  0 U l!        U RE                  5         g s  snf )NrH   rs   r9   r   r   r6   rt   rx   Fry   )#rK   rL   rn   r{   rm   r   rp   r|   r}   r~   resolution2r   N2ra   r   r   ro   r   r   r   r;   r   r   r   r   r   r   r   r   r   r   r   r   rj   r   )rO   rl   rm   rn   ro   rp   r   rq   rI   rJ   rP   r   r   rQ   s                rR   rL   Attention2dDownsample.__init__  s    /"_
#J/ DOO!LOq$))AE"2O!LM#dooa&88""1%(8(8(;;Z)*j*+i7$~#\\DNN*!#0R0#1++#ww0R0`aRVRYRY`]_`;TWWdllA<<	 "U[[FF-Qb-Q R!KK$&&)&

K 	 	

 %'! 	7 "Ms   #Ic                 f   > [         TU ]  U5        U(       a  U R                  (       a  0 U l        g g g rV   r   r   s     rR   r   Attention2dDownsample.train2  r   rT   r   c                 v    [         R                  R                  U R                  5        U R	                  5         gr   r   r   s    rR   r   &Attention2dDownsample.reset_parameters8  r   rT   c                    [         R                  " [        [         R                  " U R                  S   U[         R
                  S9[         R                  " U R                  S   U[         R
                  S95      5      R                  S5      n[         R                  " [        [         R                  " SU R                  S   SU[         R
                  S9[         R                  " SU R                  S   SU[         R
                  S95      5      R                  S5      nUSSS2S4   USSSS24   -
  R                  5       nUS   U R                  S   -  US   -   nU$ )r   r   rH   r   r9   )steprI   rJ   .Nr   )rO   rI   k_posq_posr   s        rR   r   2Attention2dDownsample._compute_attention_bias_idxs=  s   FLL+F%**MLL+F%**M
  71: 	 FLLDOOA.QvUZZXLLDOOA.QvUZZX
  71: 	 a&sD!|)<<AAC1: 22gaj@rT   c                     U R                   R                  U R                  U R                   R                  S95        0 U l        gr   r   r   s    rR   r   #Attention2dDownsample._init_buffersK  r   rT   c                 $    U R                  5         gr   r   r   s    rR   r   1Attention2dDownsample.init_non_persistent_buffersR  r   rT   rI   c                 J   [         R                  R                  5       (       d  U R                  (       a  U R                  S S 2U R
                  4   $ [        U5      nX R                  ;  a*  U R                  S S 2U R
                  4   U R                  U'   U R                  U   $ rV   r   r   s      rR   r   *Attention2dDownsample.get_attention_biasesV  r   rT   c                 T   UR                   u  p#pEU R                  U5      R                  X R                  SU R                  5      R                  SSSS5      nU R                  U5      R                  X R                  SU R                  5      R                  SSSS5      nU R                  U5      nU R                  U5      n	UR                  X R                  SU R                  5      R                  SSSS5      nXg-  U R                  -  n
XR                  UR                  5      -   n
U
R                  SS9n
X-  R                  SS5      nUR                  X R                  U R                   S   U R                   S   5      U	-   nU R#                  U5      nU R%                  U5      nU$ r   )r   r   r   rn   r   r   r   r   r   r   r{   r   rI   r   r   r   r   r   r   r   s              rR   rY   Attention2dDownsample.forward_  sY   WW
aFF1IaTWW=EEaAqQFF1IaTVV<DDQ1aPFF1I,,q/IIaTVV4<<Q1aH#//99|||#X  A&IIa$"2"21"5t7G7G7JKgUHHQKIIaLrT   )r   r   r   rj   r   ro   r   r   r   rm   rn   r   r   r   rp   r   r{   r   r   r   r   rV   )r]   r^   r_   r`   r   rb   r   r   r   r   r   ra   r	   r   r   r   r   rL   r   r   r   r   r   r   rI   r   rY   rd   re   rf   s   @rR   r   r      s    sELL011 67%))+. .  .  	. 
 .  c5c?23.  c].  BII.  . ` ]]_+ +

'95<< 9ELL 9 rT   r   c                     ^  \ rS rSrSSSSS\R
                  \R                  SS4	S\S	\S
\\\	\\4   4   S\\\	\\4   4   S\\\	\\4   4   S\\\	\\4   4   S\
S\\R                     S\\\R                        4U 4S jjjrS rSrU =r$ )
Downsampleis  r6   r9   r   rk   FNin_chsout_chsr?   r@   rA   rp   use_attnrq   rE   c                   > XS.n[         TU ]  5         [        U5      n[        U5      n[        U5      nU	=(       d    [        R                  " 5       n	[        UU4UUUU	S.UD6U l        U(       a  [        SUUUUS.UD6U l        g S U l        g )NrH   )r?   r@   rA   rE   )rl   r   rp   rq    )	rK   rL   r   r   Identityr;   rM   r   r   )rO   r  r  r?   r@   rA   rp   r  rq   rE   rI   rJ   rP   rQ   s                rR   rL   Downsample.__init__t  s     /,6"G$02;;=

 $!
 
	 - %#	
 DI DIrT   c                 j    U R                  U5      nU R                  b  U R                  U5      U-   $ U$ rV   )rM   r   )rO   rX   outs      rR   rY   Downsample.forward  s1    iil99 99Q<#%%
rT   )r   rM   )r]   r^   r_   r`   r   r   BatchNorm2dra   r	   r   rc   r   r   r   rL   rY   rd   re   rf   s   @rR   r
  r
  s  s    
 89233467")+46NN(( ( sE#s(O34	(
 #uS#X./( 3c3h/0( c5c?23( ( BII( !bii1( (T rT   r
  c                      ^  \ rS rSrSrSS\R                  \R                  SSSS4S\S\	\   S\	\   S	\
\R                     S
\
\R                     S\S\4U 4S jjjrS rSrU =r$ )ConvMlpWithNormi  zT
Implementation of MLP with 1*1 convolutions.
Input: tensor with shape [B, C, H, W]
N        Fin_featureshidden_featuresout_featuresrq   rE   dropmid_convc
                   > XS.n
[         TU ]  5         U=(       d    UnU=(       d    Un[        UUS4SUUS.U
D6U l        U(       a  [        UUS4USUUS.U
D6U l        O[
        R                  " 5       U l        [
        R                  " U5      U l        [        X#S4SU0U
D6U l
        [
        R                  " U5      U l        g )NrH   r   T)rD   rE   rq   r6   )rC   rD   rE   rq   rE   )rK   rL   r   fc1midr   r  Dropoutdrop1r;   fc2drop2)rO   r  r  r  rq   rE   r  r  rI   rJ   rP   rQ   s              rR   rL   ConvMlpWithNorm.__init__  s     /#2{)8[
 !
 
 "	 '%#	 	DH {{}DHZZ%
O1ZZWYZZZ%
rT   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU$ rV   )r  r   r"  r#  r$  rW   s     rR   rY   ConvMlpWithNorm.forward  sH    HHQKHHQKJJqMHHQKJJqMrT   )r"  r$  r  r#  r   )r]   r^   r_   r`   __doc__r   r   r  ra   r   r   r   floatrc   rL   rY   rd   re   rf   s   @rR   r  r    s     .2*.)+*,.."(&(& &c](& #3-	(&
 BII(& RYY(& (& (& (&T rT   r  c                      ^  \ rS rSrS\R
                  \R                  SSSSSSSS4S\S	\S
\	\R                     S\	\R                     S\S\S\\   S\\\\\4   4   S\\   S\4U 4S jjjrS rSrU =r$ )EfficientFormerV2Blocki        @r  h㈵>rk   NTrl   	mlp_ratiorq   rE   	proj_drop	drop_pathlayer_scale_init_valuerp   r@   r  c           
      .  > XS.n[         TU ]  5         U
(       aj  [        U4UUU	S.UD6U l        Ub  [	        X40 UD6O[
        R                  " 5       U l        US:  a  [        U5      O[
        R                  " 5       U l	        OS U l        S U l        S U l	        [        SU[        X-  5      UUUSS.UD6U l        Ub  [	        X40 UD6O[
        R                  " 5       U l        US:  a  [        U5      U l        g [
        R                  " 5       U l        g )NrH   )rp   rq   r@   r  T)r  r  rq   rE   r  r  r  )rK   rL   rh   token_mixerr   r   r  ls1r   
drop_path1r  ra   mlpls2
drop_path2)rO   rl   r.  rq   rE   r/  r0  r1  rp   r@   r  rI   rJ   rP   rQ   s                 rR   rL   EfficientFormerV2Block.__init__  s'    /* %#	 
  D 7M6X $3/13^`^i^i^k H5>^hy1DO#DDH"DO" 
0!
 
 3I2T  /+-/Z\ZeZeZg 	1:R(9-R[[]rT   c                     U R                   b1  XR                  U R                  U R                  U5      5      5      -   nXR                  U R	                  U R                  U5      5      5      -   nU$ rV   )r3  r5  r4  r8  r7  r6  rW   s     rR   rY   EfficientFormerV2Block.forward  sZ    'OODHHT-=-=a-@$ABBA! 566rT   )r5  r8  r4  r7  r6  r3  )r]   r^   r_   r`   r   r   r  ra   r)  r   r   r   r	   r   rc   rL   rY   rd   re   rf   s   @rR   r+  r+    s      ")+*,..!!6:67$(!-S-S -S BII	-S
 RYY-S -S -S %-UO-S c5c?23-S SM-S -S -S^ rT   r+  c            
          ^  \ rS rSr\R
                  \R                  SS4S\S\S\\R                     S\\R                     4U 4S jjjr
SrU =r$ )	Stem4i  Nr  r  rq   rE   c           
         > XVS.n[         TU ]  5         SU l        [        UUS-  4SSSSUUS.UD6U l        [        US-  U4SSSSUUS.UD6U l        g )NrH   r3   r9   r6   r   T)r?   r@   rA   rD   rE   rq   )rK   rL   r@   r   conv1conv2)	rO   r  r  rq   rE   rI   rJ   rP   rQ   s	           rR   rL   Stem4.__init__  s     / qL	
 a!	
 	

 !qL

 !

 


rT   )r?  r@  r@   )r]   r^   r_   r`   r   r   r  ra   r   r   rL   rd   re   rf   s   @rR   r=  r=    sY    
 *,*,.. 
 
  
 BII	 

 RYY 
  
rT   r=  c                    .  ^  \ rS rSrSSSSSSSSSS	\R
                  \R                  SS4S
\S\S\S\\\	\\4   4   S\
S\\   S\
S\
S\S\\\	\S4   4   S\S\\\\   4   S\\   S\\R                     S\\R                     4U 4S jjjrS rSrU =r$ )EfficientFormerV2Stagei9  rk   TNFr   r,  r  r-  rl   dim_outdepthrp   
downsampleblock_stridedownsample_use_attnblock_use_attnnum_vitr.  .r/  r0  r1  rq   rE   c                 (  > UUS.n[         TU ]  5         SU l        [        U5      " U
5      n
[	        U5      nU(       aM  [        UU4UUUUS.UD6U l        Un[        U Vs/ s H  n[        R                  " US-  5      PM     sn5      nO!X:X  d   e[        R                  " 5       U l        / n[        U5       H8  nX9-
  S-
  n[        U4UUU
U   U=(       a    UU:  UUU   UUUS.	UD6nUU/-  nM:     [        R                  " U6 U l        g s  snf )NrH   F)r  rp   rE   rq   r9   r   )	rp   r@   r.  r  r/  r0  r1  rq   rE   )rK   rL   grad_checkpointingr   r   r
  rF  r|   r}   r~   r   r  ranger+  
Sequentialblocks)rO   rl   rD  rE  rp   rF  rG  rH  rI  rJ  r.  r/  r0  r1  rq   rE   rI   rJ   rP   r   rO  	block_idx
remain_idxbrQ   s                           rR   rL   EfficientFormerV2Stage.__init__;  s@   ( /"'e$Y/	z*
( -%%# DO C*E*Q		!a% 0*EFJ>!> kkmDOuI1,J&%##I.'BI
,B##I.'=#% A qcMF &  mmV,-  Fs   !#Dc                     U R                  U5      nU R                  (       a;  [        R                  R	                  5       (       d  [        U R                  U5      nU$ U R                  U5      nU$ rV   )rF  rL  r   r   is_scriptingr   rO  rW   s     rR   rY   EfficientFormerV2Stage.forwardx  sV    OOA""599+A+A+C+Ct{{A.A  AArT   )rO  rF  rL  )r]   r^   r_   r`   r   r   r  ra   r	   r   rc   r   r)  r   r   r   rL   rY   rd   re   rf   s   @rR   rC  rC  9  s+    78#*.(-#(9;!356:)+*,..%;-;- ;- 	;-
 c5c?23;- ;- #3-;- "&;- !;- ;- UE%*$556;- ;- UDK/0;- %-UO;- BII;-  RYY!;- ;-z rT   rC  c            #         ^  \ rS rSr                  S.S\\S4   S\S\\\\\4   4   S\S\\\S4      S\\\	S4      S	\\
\\
S4   \\\
S4   S4   4   S
\S\
S\S\S\
S\
S\
S\\
   S\S\	4"U 4S jjjrS/S\	4S jjrS/S\	4S jjr\R                  R                   S 5       r\R                  R                   S0S j5       r\R                  R                   S/S j5       r\R                  R                   S\R*                  4S j5       rS1S\S\\   4S jjr\R                  R                   S/S j5       r     S2S\R2                  S \\\\\   4      S!\	S"\	S#\S$\	S\\\R2                     \\R2                  \\R2                     4   4   4S% jjr   S3S \\\\   4   S&\	S'\	4S( jjrS) rS0S*\	4S+ jjrS, rS-r U =r!$ )4r   i  depths.in_chansimg_sizeglobal_pool
embed_dimsdownsamples
mlp_ratiosrE   norm_epsrq   num_classes	drop_rateproj_drop_ratedrop_path_rater1  rJ  distillationc                   > [         TU ]  5         UUS.nUS;   d   eXl        X l        X@l        / U l        [        U5      n[        [        U5      U	S9n[        U
5      n
[        X%S   4XS.UD6U l        US   nSn[        U5      n[        XSS9nU=(       d    S	S
[        U5      S-
  -  -   n[        U5      " U5      n/ n[        U5       H  n[!        U Vs/ s H  n["        R$                  " UU-  5      PM     sn5      n['        UUU   4UU   UUU   US:X  a  SOS US:  US:  UUU   UUU   UU
US.UD6nUU   (       a  US-  nUU   nU =R
                  [)        UUSU 3S9/-  sl        UR+                  U5        M     [,        R.                  " U6 U l        US   =U l        U l        U" US   40 UD6U l        [,        R8                  " U5      U l        US:  a  [,        R<                  " US   U40 UD6O[,        R>                  " 5       U l         UU l!        U RB                  (       a<  US:  a  [,        R<                  " US   U40 UD6O[,        R>                  " 5       U l"        OS U l"        U RG                  SS9  SU l$        g s  snf )NrH   )avgr[   )epsr   )rq   rE   r3   T)	stagewiseFr   r   r9   r6   )rE  rp   rF  rG  rH  rI  rJ  r.  r/  r0  r1  rq   rE   zstages.)num_chs	reductionmoduler   Fneeds_reset)%rK   rL   r`  rY  r[  feature_infor   r   r   r   r=  stemlenr   r   rM  r|   r}   r~   rC  dictappendr   rN  stagesnum_featureshead_hidden_sizenormr!  	head_dropLinearr  headdist	head_distinit_weightsdistilled_training)rO   rX  rY  rZ  r[  r\  r]  r^  rE   r_  rq   r`  ra  rb  rc  r1  rJ  rd  rI   rJ   rP   prev_dimr@   
num_stagesdprrt  iscurr_resolutionstagerQ   s                                 rR   rL   EfficientFormerV2.__init__  s   , 	/k)))& &X&^J7XF
!),	(qMdYdacd	a=[
'$O!KX3v;?0K%Kz*:6
z"A#H$MHqTYYq6z%:H$MNO*1 Qi*&q>"#q&Qd$%F Av$Q-(a&'=#%  !E$ 1~!!!}H$x6T[\][^R_"`!aaMM% 1 #2 mmV, 5?rNBD1z"~44	I.DORSOBIIjnk@R@Y[YdYdYf	 	99MX[\_RYYz"~{IbIbdbmbmboDN!DN 	e,"'Q %Ns   #J
rn  c                 0   [        U[        R                  5      (       aM  [        UR                  SS9  UR
                  b+  [        R                  R                  UR
                  S5        g g U(       a#  [        US5      (       a  UR                  5         g g g )N{Gz?)stdr   r   )

isinstancer   ry  r   weightrD   r   	constant_hasattrr   )rO   mrn  s      rR   _init_weightsEfficientFormerV2._init_weights  sk    a##!((,vv!!!!&&!, "WQ(:;;  <[rT   c                 J    U R                  [        U R                  US95        g )Nrm  )applyr   r  )rO   rn  s     rR   r}  EfficientFormerV2.init_weights  s    

74--;GHrT   c                 h    U R                  5        VVs1 s H  u  pSU;   d  M  UiM     snn$ s  snnf )Nr   )named_parameters)rO   r   _s      rR   no_weight_decay!EfficientFormerV2.no_weight_decay  s/    "335Q5da9Kq9P5QQQs   ..c                     [        SSS/S9nU$ )Nz^stem)z^stages\.(\d+)N)z^norm)i )rp  rO  )rr  )rO   coarsematchers      rR   group_matcherEfficientFormerV2.group_matcher  s    -/CD
 rT   c                 6    U R                    H	  nXl        M     g rV   )rt  rL  )rO   enabler  s      rR   set_grad_checkpointing(EfficientFormerV2.set_grad_checkpointing  s    A#)  rT   r   c                 2    U R                   U R                  4$ rV   rz  r|  r   s    rR   get_classifier EfficientFormerV2.get_classifier  s    yy$..((rT   c                 2   Xl         Ub  X l        US:  a!  [        R                  " U R                  U5      O[        R
                  " 5       U l        US:  a'  [        R                  " U R                  U5      U l        g [        R
                  " 5       U l        g )Nr   )r`  r[  r   ry  ru  r  rz  r|  )rO   r`  r[  s      rR   reset_classifier"EfficientFormerV2.reset_classifier  sm    &"*ALqBIId//=VXVaVaVc	FQTUo4#4#4kB[][f[f[hrT   c                     Xl         g rV   )r~  )rO   r  s     rR   set_distilled_training(EfficientFormerV2.set_distilled_training  s    "(rT   rX   indicesrw  
stop_early
output_fmtintermediates_onlyc                 X   US;   d   S5       e/ n[        [        U R                  5      U5      u  pU R                  U5      n[        U R                  5      S-
  n
[        R
                  R                  5       (       d  U(       d  U R                  nOU R                  SU	S-    n[        U5       HW  u  pU" U5      nX;   d  M  X:X  a-  U(       a  U R                  U5      OUnUR                  U5        MF  UR                  U5        MY     U(       a  U$ WU
:X  a  U R                  U5      nX4$ )a  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    norm: Apply norm layer to compatible intermediates
    stop_early: Stop iterating over blocks when last desired intermediate hit
    output_fmt: Shape of intermediate feature outputs
    intermediates_only: Only return intermediate features
Returns:

)NCHWzOutput shape must be NCHW.r   N)
r   rq  rt  rp  r   r   rU  	enumeraterw  rs  )rO   rX   r  rw  r  r  r  intermediatestake_indices	max_indexlast_idxrt  feat_idxr  x_inters                  rR   forward_intermediates'EfficientFormerV2.forward_intermediates  s   * Y&D(DD&"6s4;;7G"Q IIaLt{{#a'99!!##:[[F[[)a-0F(0OHaA''.2diilG!((1!((+  1   x		!ArT   
prune_norm
prune_headc                     [        [        U R                  5      U5      u  pEU R                  SUS-    U l        U(       a  [        R                  " 5       U l        U(       a  U R                  SS5        U$ )z?Prune layers not required for specified intermediates.
        Nr   r   r[   )r   rq  rt  r   r  rw  r  )rO   r  r  r  r  r  s         rR   prune_intermediate_layers+EfficientFormerV2.prune_intermediate_layers5  s[     #7s4;;7G"Qkk.9q=1DI!!!R(rT   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rV   )rp  rt  rw  rW   s     rR   forward_features"EfficientFormerV2.forward_featuresE  s.    IIaLKKNIIaLrT   
pre_logitsc                 T   U R                   S:X  a  UR                  SS9nU R                  U5      nU(       a  U$ U R                  U5      U R	                  U5      p1U R
                  (       a7  U R                  (       a&  [        R                  R                  5       (       d  X4$ X-   S-  $ )Nrf  )r9   r6   r   r9   )
r[  meanrx  rz  r|  r~  r   r   r   rU  )rO   rX   r  x_dists       rR   forward_headEfficientFormerV2.forward_headK  s    u$6"ANN1HIIaL$.."36""t}}UYY=S=S=U=U9 J!##rT   c                 J    U R                  U5      nU R                  U5      nU$ rV   )r  r  rW   s     rR   rY   EfficientFormerV2.forwardY  s'    !!!$a rT   )r{  r~  ro  r[  rz  r|  rx  rv  rY  rw  r`  ru  rt  rp  )r6   r)   rf  NNr3   r\   r-  gelu  r  r  r  r-  r   TNNr   ri  rV   )NFFr  F)r   FT)"r]   r^   r_   r`   r   ra   r	   rb   r   rc   r)  rL   r  r}  r   r   ignorer  r  r  r   r   r  r  r  r   r   r  r  r  r  rY   rd   re   rf   s   @rR   r   r     s)    47$486:YZ+"##!$&$&6:!%)S(#s(OS( S( CsCx01	S(
 S( !sCx1S( "%c	"23S( eU5#:%6eE3J>OQT>T8UUVS( S( S( S( S( S( "S( "S(  %-UO!S(" #S($ %S( S(j!D !I I YYR R YY  YY* * YY)		 ) )iC ihsm i YY) ) 8<$$',1 ||1  eCcN341  	1 
 1  1  !%1  
tELL!5tELL7I)I#JJ	K1 j ./$#	3S	>*  	 $$ $ rT   c                 4    U SSS SSS[         [        SSSS	.UE$ )
Nr  )r6   r)   r)   Tgffffff?bicubicr  zstem.conv1.convz
apache-2.0)urlr`  
input_size	pool_sizefixed_input_sizecrop_pctinterpolationr  r  
classifier
first_convlicenser
   )r  kwargss     rR   _cfgr  _  s7    =tae)%.B+;L  rT   ztimm/)	hf_hub_id)z#efficientformerv2_s0.snap_dist_in1kz#efficientformerv2_s1.snap_dist_in1kz#efficientformerv2_s2.snap_dist_in1kz"efficientformerv2_l.snap_dist_in1kc           	      `    UR                  SS5      n[        [        X4S[        SUS90UD6nU$ )Nout_indices)r   r   r9   r6   feature_cfgT)flatten_sequentialr  )popr   r   rr  )variant
pretrainedr  r  models        rR   _create_efficientformerv2r  {  sA    **]L9K 7DkJ E LrT   r   c           	      t    [        [        S   [        S   SS[        S   S9n[	        SSU 0[        U40 UD6D6$ )Nr/   r9   r  rX  r\  rJ  rc  r^  r  )efficientformerv2_s0rr  EfficientFormer_depthEfficientFormer_width EfficientFormer_expansion_ratiosr  r  r  
model_argss      rR   r  r    L    $T*(.3D9J %q
qVZ[eVpioVpqqrT   c           	      t    [        [        S   [        S   SS[        S   S9n[	        SSU 0[        U40 UD6D6$ )Nr.   r9   r  r  r  )efficientformerv2_s1r  r  s      rR   r  r    r  rT   c           	      t    [        [        S   [        S   SS[        S   S9n[	        SSU 0[        U40 UD6D6$ )Nr-   r3   r  r  r  )efficientformerv2_s2r  r  s      rR   r  r    sL    $T*(.3D9J %q
qVZ[eVpioVpqqrT   c           	      t    [        [        S   [        S   SS[        S   S9n[	        SSU 0[        U40 UD6D6$ )Nr,   r8   g?r  r  )efficientformerv2_lr  r  s      rR   r  r    sL    $S)(-3C8J %pzpUYZdUohnUopprT   )r[   ri  )>r(  r}   	functoolsr   typingr   r   r   r   r   r	   r   torch.nnr   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r  r  r  r   r;   rh   r   r   r
  r  r+  rN  r=  rC  r   r  default_cfgsr  r  r  r  r  r  rT   rR   <module>r      s      ; ;   A    + + ' < 
 



	  



	  
_
P
A
2	$  #ryy #L{%((// {|uxx ,qEHHOO qh/ /d6bii 6r4RYY 4n!
BMM !
HERYY EP[		 [|	 %+/, ,0, ,0, +/+&   r8I r r r8I r r r8I r r q7H q qrT   