
    RЦi                     "   S r SS/rSSKJrJrJrJrJr  SSKJ	r	  SSK
r
SSKJr  SSKJs  Jr  SSKJrJr  SSKJrJrJr  S	S
KJr  S	SKJr  S	SKJr  S	SKJr  S	SKJ r J!r!  SdS\"=(       d    \#=(       d    \$4S jjr%SeS\"=(       d    \#=(       d    \$S\&S\&4S jjr'S\&=(       d    \#\&S4   S\&=(       d    \#\&S4   4S jr( " S S\RR                  5      r* " S S\RR                  5      r+ " S S\RR                  5      r, " S S\RR                  5      r- " S  S!\RR                  5      r. " S" S#\RR                  5      r/\" \/5         " S$ S%\RR                  5      r0 " S& S'\RR                  5      r1    SfS(\&S)\&S*\&S+\2S,\3S-\3S.\4S/\34S0 jjr5 " S1 S2\Rl                  5      r7 " S3 S4\RR                  5      r8 " S5 S6\RR                  5      r9 " S7 S8\RR                  5      r: " S9 S\RR                  5      r; " S: S\RR                  5      r<SgS; jr=\!" 0 S<\=" S=S>9_S?\=" S=S>9_S@\=" S=SASBSCSD9_SE\=" S=SFSGSCSD9_SH\=" S=S>9_SI\=" S=SASBSCSD9_SJ\=" S=SFSGSCSD9_SK\=" S=S>9_SL\=" S=SASBSCSD9_SM\=" S=SFSGSCSD9_SN\=" S=SCSO9_SP\=" S=SCSO9_SQ\=" S=SASBSCSD9_SR\=" S=SFSGSCSD9_SS\=" S=STSUSCSD9_SV\=" S=SCSO9_SW\=" S=SASBSCSD9_\=" S=SXSYSCSD9\=" S=STSUSCSD9SZ.E5      r>ShS[ jr?ShS\ jr@\ ShS] j5       rA\ ShS^ j5       rB\ ShS_ j5       rC\ ShS` j5       rD\ ShSa j5       rE\ ShSb j5       rF\ ShSc j5       rGg)ia   EfficientViT (by MIT Song Han's Lab)

Paper: `Efficientvit: Enhanced linear attention for high-resolution low-computation visual recognition`
    - https://arxiv.org/abs/2205.14756

Adapted from official impl at https://github.com/mit-han-lab/efficientvit
EfficientVitEfficientVitLarge    )ListOptionalTupleTypeUnion)partialNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)SelectAdaptivePool2dcreate_conv2dGELUTanh   )build_model_with_cfg)feature_take_indices)register_notrace_module)checkpoint_seq)register_modelgenerate_default_cfgsxc                     [        U [        [        45      (       a  [        U 5      $ [        U5       Vs/ s H  o PM     sn$ s  snf N)
isinstancelisttuplerange)r   repeat_time_s      [/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/efficientvit_mit.pyval2listr"      s;    !dE]##Aw[)*)!A)***s   Amin_len
idx_repeatc                     [        U 5      n [        U 5      S:  a,  [        U[        U 5      -
  5       Vs/ s H  o0U   PM	     snXU& [        U 5      $ s  snf )Nr   )r"   lenr   r   )r   r#   r$   r    s       r!   	val2tupler'       sP    A
1vz;@3q6AQ;R#S;RajM;R#SZ 8O $Ts   Akernel_size.returnc                     [        U [        5      (       a%  [        U  Vs/ s H  n[        U5      PM     sn5      $ U S-  S:  d   S5       eU S-  $ s  snf )N   r   z kernel size should be odd number)r   r   get_same_padding)r(   kss     r!   r,   r,   )   sX    +u%%[A[r&r*[ABBQ"F$FF"a Bs   Ac                      ^  \ rS rSrSSSSSS\R
                  \R                  SS4
S\S\S	\\\	\\4   4   S
\S\S\S\
S\S\\\R                        S\\\R                        4U 4S jjjrS rSrU =r$ )ConvNormAct1      r   F        Nin_channelsout_channelsr(   stridedilationgroupsbiasdropout
norm_layer	act_layerc           	      6  > XS.n[         TU ]  5         [        R                  " USS9U l        [        UU4UUUUUS.UD6U l        U	(       a
  U	" SSU0UD6O[        R                  " 5       U l        U
b  U
" SS9U l	        g [        R                  " 5       U l	        g )NdevicedtypeFinplace)r(   r5   r6   r7   r8   num_featuresT )
super__init__nnDropoutr9   r   convIdentitynormact)selfr3   r4   r(   r5   r6   r7   r8   r9   r:   r;   r>   r?   dd	__class__s                 r!   rE   ConvNormAct.__init__2   s     /zz'59!	
 $	
 	
	 DNJ?L?B?SUS^S^S`	.7.C9T*    c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ r   )r9   rH   rJ   rK   rL   r   s     r!   forwardConvNormAct.forwardQ   s;    LLOIIaLIIaLHHQKrP   )rK   rH   r9   rJ   )__name__
__module____qualname____firstlineno__rF   BatchNorm2dReLUintr	   r   boolfloatr   r   ModulerE   rS   __static_attributes____classcell__rN   s   @r!   r/   r/   1   s    
 8946NN3577WW W sE#s(O34	W
 W W W W W !bii1W  RYY0W W> rP   r/   c                   <  ^  \ rS rSrSSS\R
                  \R                  S4SS4S\S\S\S	\S
\\	\
\	\	4   4   S\\\R                     \
\\\R                        S4   4   S\\\R                     \
\\\R                        S4   4   4U 4S jjjrS rSrU =r$ )DSConvY   r1   r   FNr3   r4   r(   r5   use_biasr:   .r;   c
                    > XS.n
[         TU ]  5         [        US5      n[        US5      n[        US5      n[        UUUU4UUS   US   US   S.U
D6U l        [        UUS4US   US   US   S.U
D6U l        g )Nr=   r+   r   )r7   r:   r;   r8   r   r:   r;   r8   )rD   rE   r'   r/   
depth_conv
point_conv)rL   r3   r4   r(   r5   re   r:   r;   r>   r?   rM   rN   s              r!   rE   DSConv.__init__Z   s     /Xq)z1-
i+	%	


 !!}l!

 

 &
 "!}l!
 
rP   c                 J    U R                  U5      nU R                  U5      nU$ r   rh   ri   rR   s     r!   rS   DSConv.forward   s#    OOAOOArP   rl   )rU   rV   rW   rX   rF   rY   ReLU6r[   r	   r\   r   r   r^   r   rE   rS   r_   r`   ra   s   @r!   rc   rc   Y   s    
  !7<XZXfXfXZX`X`bfWg%
%
 %
 	%

 %
 D%d
"334%
 d299ouXd299o5NPS5S/TTU%
 T"))_eHT"))_4Ms4R.SST%
 %
N rP   rc   c                   N  ^  \ rS rSrSSSSS\R
                  \R                  S4SS4	S\S\S\S	\S
\\   S\	S\
\\\\4   4   S\
\\R                     \\\\R                        S4   4   S\
\\R                     \\\\R                        S4   4   4U 4S jjjrS rSrU =r$ )	ConvBlock   r1   r   NFr3   r4   r(   r5   mid_channelsexpand_ratiore   r:   .r;   c                 "  > XS.n[         TU ]  5         [        US5      n[        US5      n[        U	S5      n	U=(       d    [        X-  5      n[	        UUUU4US   U	S   US   S.UD6U l        [	        UUUS4US   U	S   US   S.UD6U l        g )Nr=   r+   r   rg   r   )rD   rE   r'   roundr/   conv1conv2rL   r3   r4   r(   r5   rr   rs   re   r:   r;   r>   r?   rM   rN   s                r!   rE   ConvBlock.__init__   s     /Xq)z1-
i+	#Hu[-G'H 		

 "!}l!	
 	

 !		

 "!}l!	
 	

rP   c                 J    U R                  U5      nU R                  U5      nU$ r   rv   rw   rR   s     r!   rS   ConvBlock.forward   s!    JJqMJJqMrP   r{   rU   rV   rW   rX   rF   rY   rn   r[   r   r]   r	   r\   r   r   r^   rE   rS   r_   r`   ra   s   @r!   rp   rp      s	   
  !*."#7<XZXfXfXZX`X`bfWg(
(
 (
 	(

 (
 #3-(
  (
 D%d
"334(
 d299ouXd299o5NPS5S/TTU(
 T"))_eHT"))_4Ms4R.SST(
 (
T rP   rp   c                   d  ^  \ rS rSrSSSSS\R
                  \R                  \R                  S4SS4	S\S\S	\S
\S\\   S\	S\
\\\S4   4   S\
\\R                     \\\\R                        S4   4   S\
\\R                     \\\\R                        S4   4   4U 4S jjjrS rSrU =r$ )MBConv   r1   r   N   Fr3   r4   r(   r5   rr   rs   re   .r:   r;   c           	      d  > XS.n[         TU ]  5         [        US5      n[        US5      n[        U	S5      n	U=(       d    [        X-  5      n[	        UUS4SUS   U	S   US   S.UD6U l        [	        UUU4UUUS   U	S   US   S.UD6U l        [	        UUS4US   U	S   US   S.UD6U l        g )	Nr=   r1   r   r   )r5   r:   r;   r8   r5   r7   r:   r;   r8   r+   rg   )rD   rE   r'   ru   r/   inverted_convrh   ri   rx   s                r!   rE   MBConv.__init__   s    /Xq)z1-
i+	#Hu[-G'H(	
 !!}l!	
 	
 &

 !!}l!

 

 &
 "!}l!
 
rP   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ r   )r   rh   ri   rR   s     r!   rS   MBConv.forward   s3    q!OOAOOArP   )rh   r   ri   r}   ra   s   @r!   r   r      s   
  !*."#6;XZXfXfXZX`X`bdbjbjlpWq2
2
 2
 	2

 2
 #3-2
  2
 D%c	"2232
 d299ouXd299o5NPS5S/TTU2
 T"))_eHT"))_4Ms4R.SST2
 2
h rP   r   c                   T  ^  \ rS rSrSSSSSS\R
                  \R                  S4SS4
S\S\S	\S
\S\\   S\	S\S\
\\\S4   4   S\
\\R                     \\\\R                        S4   4   S\
\\R                     \\\\R                        S4   4   4U 4S jjjrS rSrU =r$ )FusedMBConv   r1   r   Nr   Fr3   r4   r(   r5   rr   rs   r7   re   .r:   r;   c           	      "  > XS.n[         TU ]  5         [        US5      n[        U	S5      n	[        U
S5      n
U=(       d    [        X-  5      n[	        UUU4UUU	S   U
S   US   S.UD6U l        [	        UUS4U	S   U
S   US   S.UD6U l        g )Nr=   r+   r   r   r   rg   )rD   rE   r'   ru   r/   spatial_convri   )rL   r3   r4   r(   r5   rr   rs   r7   re   r:   r;   r>   r?   rM   rN   s                 r!   rE   FusedMBConv.__init__   s     /Xq)z1-
i+	#Hu[-G'H'

 !!}l!

 

 &
 "!}l!
 
rP   c                 J    U R                  U5      nU R                  U5      nU$ r   )r   ri   rR   s     r!   rS   FusedMBConv.forward   s%    a OOArP   )ri   r   r}   ra   s   @r!   r   r      s   
  !*."#6;XZXfXfXZX`X`bfWg)
)
 )
 	)

 )
 #3-)
  )
 )
 D%c	"223)
 d299ouXd299o5NPS5S/TTU)
 T"))_eHT"))_4Ms4R.SST)
 )
V rP   r   c                     ^  \ rS rSrSrSSSSS\R                  4S\R                  SS	SS4S
\S\S\	\   S\
S\S\\\\S4   4   S\\\R                     \\	\\R                        S4   4   S\\\R                     \\	\\R                        S4   4   S\\R                     S\\S4   S\
4U 4S jjjrS rS rSrU =r$ )LiteMLAi&  z(Lightweight multi-scale linear attentionN      ?   FNN)   h㈵>r3   r4   headsheads_ratiodimre   .r:   r;   kernel_funcscalesepsc                   > XS.n[         TU ]  5         Xl        U=(       d    [        X-  U-  5      nX5-  n[	        US5      n[	        US5      n[	        US5      nXPl        [        USU-  S4US   US   US   S.UD6U l        [        R                  " U
 Vs/ s Hp  n[        R                  " [        R                  " SU-  SU-  U4[        U5      SU-  US   S.UD6[        R                  " SU-  SU-  S4SU-  US   S.UD65      PMr     sn5      U l        U	" S	S
9U l        [        US[        U
5      -   -  US4US   US   US   S.UD6U l        g s  snf )Nr=   r+   r1   r   r   )r8   r:   r;   )paddingr7   r8   )r7   r8   Fr@   )rD   rE   r   r[   r'   r   r/   qkvrF   
ModuleList
SequentialConv2dr,   aggregr   r&   proj)rL   r3   r4   r   r   r   re   r:   r;   r   r   r   r>   r?   rM   	total_dimscalerN   s                    r!   rE   LiteMLA.__init__)  s     />[/+=>K	Xq)z1-
i+		M
 !!!}l
 
 mm  %
   MM			M	M -U3y=!!  		!i-Yd!e)RZ[\R]dacd  %
  'u5S[)
 !!!}l
 
	#%
s   A7Ec                    UR                   nUR                  5       UR                  5       UR                  5       p2nUR                  SS5      U-  nX-  nUSS S24   USSS 24   U R                  -   -  nUR	                  U5      $ )N.)r?   r]   	transposer   to)rL   qkvr?   kvouts          r!   _attnLiteMLA._attng  s{    '')QWWY	a[[R 1$f#ss(ms38}txx78vve}rP   c                 F   UR                   u  p#pEU R                  U5      nU/nU R                   H  nUR                  U" U5      5        M     [        R
                  " USS9nUR                  USSU R                  -  XE-  5      R                  SS5      nUR                  SSS9u  pnU R                  U	5      n	U R                  U
5      n
[        R                  " USSSS	9n[        R                  R                  5       (       dE  [        R                  " UR                   R"                  S
S9   U R%                  XU5      nS S S 5        OU R%                  XU5      nWR                  SS5      R                  USXE5      nU R'                  U5      nU$ ! , (       d  f       ND= f)Nr   )r   r   r1   r   )r   r   constantr   )modevalueF)device_typeenabled)shaper   r   appendtorchcatreshaper   r   chunkr   Fpadjitis_scriptingautocastr>   typer   r   )rL   r   Br    HWr   multi_scale_qkvopr   r   r   r   s                r!   rS   LiteMLA.forwardo  sd   WW
a hhqk%++B""2c7+ ))O;)11!RTXXquMWWXZ\^_!''r'2a QQEE!V*B7yy%%''AHHMM5Ijjq) JI **Q1%C mmB#++Ar18iin
 JIs   .F
F )r   r   r   r   r   r   )rU   rV   rW   rX   __doc__rF   rY   rZ   r[   r   r]   r	   r\   r   r   r^   rE   r   rS   r_   r`   ra   s   @r!   r   r   &  s9   2 $(!$6;Y]_a_m_mXnWc+-77&*<
<
 <
 C=	<

 <
 <
 D%c	"223<
 d299ouXd299o5NPS5S/TTU<
 T"))_eHT"))_4Ms4R.SST<
 bii<
 #s(O<
 <
 <
| rP   r   c                      ^  \ rS rSrSSS\R
                  \R                  SS4S\S\S\S	\S
\	\R                     S\	\R                     4U 4S jjjrS rSrU =r$ )EfficientVitBlocki  r          Nr3   r   head_dimrs   r:   r;   c	                   > XxS.n	[         T
U ]  5         [        [        SUUUUS U4S.U	D6[        R
                  " 5       5      U l        [        [        SUUUSS S U4XfS 4S.U	D6[        R
                  " 5       5      U l        g )Nr=   )r3   r4   r   r   r:   TTF)r3   r4   rs   re   r:   r;   rC   )	rD   rE   ResidualBlockr   rF   rI   context_moduler   local_module)rL   r3   r   r   rs   r:   r;   r>   r?   rM   rN   s             r!   rE   EfficientVitBlock.__init__  s     /+ '(' *-  KKM

 * '(), $
3$6  KKM
rP   c                 J    U R                  U5      nU R                  U5      nU$ r   r   r   rR   s     r!   rS   EfficientVitBlock.forward  s'    "a rP   r   )rU   rV   rW   rX   rF   rY   	Hardswishr[   r]   r   r^   rE   rS   r_   r`   ra   s   @r!   r   r     s     "%"#*,..)+#
#
 #
 	#

  #
 RYY#
 BII#
 #
J rP   r   c                      ^  \ rS rSr  SS\\R                     S\\R                     S\\R                     4U 4S jjjrS rSr	U =r
$ )	r   i  mainshortcutpre_normc                 x   > [         TU ]  5         Ub  UO[        R                  " 5       U l        Xl        X l        g r   )rD   rE   rF   rI   r   r   r   )rL   r   r   r   rN   s       r!   rE   ResidualBlock.__init__  s.     	$,$8bkkm	 rP   c                     U R                  U R                  U5      5      nU R                  b  X R                  U5      -   nU$ r   r   r   r   )rL   r   ress      r!   rS   ResidualBlock.forward  s9    iia()==$a((C
rP   r   r   )rU   rV   rW   rX   r   rF   r^   rE   rS   r_   r`   ra   s   @r!   r   r     sV     -1,0		!299%	! ryy)	! ryy)		! 	! rP   r   r3   r4   r5   rs   r:   r;   
fewer_norm
block_typec
                    XS.n
US;   d   eUS:X  a[  US:X  a*  [        S
U UUU(       a  SOSU(       a  S U4OUUS 4S.U
D6nU$ [        S
U UUU(       a  SOSU(       a  S U4OUUS 4S.U
D6n U$ US:X  a,  [        S
U UUUU(       a  SOSU(       a  S S U4OUXUS 4S	.U
D6nU$ [        S
U UUUU(       a  SOSU(       a  S U4OUUS 4S	.U
D6nU$ )Nr=   )defaultlargefusedr   r   )TFF)r3   r4   r5   re   r:   r;   r   )r3   r4   r5   rs   re   r:   r;   rC   )rc   rp   r   r   )r3   r4   r5   rs   r:   r;   r   r   r>   r?   rM   blocks               r!   build_local_blockr     sU    	+B6666q" ')*4%1;D*-$d+ ET LA  ')*4%1;D*-$d+ E@ L- " 	'))0:,7AD$
3z$6	 	E* L   	'))*4%1;D*-$d+	 	E LrP   c                      ^  \ rS rSr   S
S\S\S\S\\R                     S\\R                     S\4U 4S jjjr	S	r
U =r$ )Stemi  in_chsout_chsdepthr:   r;   r   c	                 4  > [         TU ]  5         XxS.n	SU l        U R                  S[	        UU4SSUUS.U	D65        Sn
[        U5       HK  nU R                  SU
 3[        [        S
UUSSUUUS	.U	D6[        R                  " 5       5      5        U
S-  n
MM     g )Nr=   r+   in_convr1   )r(   r5   r:   r;   r   r   r   )r3   r4   r5   rs   r:   r;   r   rC   )
rD   rE   r5   
add_moduler/   r   r   r   rF   rI   )rL   r   r   r   r:   r;   r   r>   r?   rM   
stem_blockr    rN   s               r!   rE   Stem.__init__  s     	/ %# 	
 
uAOOc*.! 	 '!(!")')	 	 1  !OJ rP   )r5   )r   NN)rU   rV   rW   rX   r[   r   rF   r^   strrE   r_   r`   ra   s   @r!   r   r     sb     (** * 	*
 RYY* BII* * *rP   r   c                      ^  \ rS rSr   SS\S\S\S\\R                     S\\R                     S\S\S	\	4U 4S
 jjjr
S rSrU =r$ )EfficientVitStagei9  r   r   r   r:   r;   rs   r   	vit_stagec                   > XS.n[         TU ]  5         [        [        SUUSUUUUS.UD6S 5      /nUnU(       a3  [	        U5       H#  nUR                  [        SUUUUUS.UD65        M%     OQ[	        SU5       HA  nUR                  [        [        SUUSUUUS.UD6[        R                  " 5       5      5        MC     [        R                  " U6 U l
        g )Nr=   r+   )r3   r4   r5   rs   r:   r;   r   r3   r   rs   r:   r;   r   )r3   r4   r5   rs   r:   r;   rC   rD   rE   r   r   r   r   r   rF   rI   r   blocks)rL   r   r   r   r:   r;   rs   r   r   r>   r?   rM   r   r    irN   s                  r!   rE   EfficientVitStage.__init__:  s    / 	"$)%#$	 	 
  5\% $*!)%1#-"+ 	 " 1e_m% $*%, %1#-"+  KKM  % mmV,rP   c                 $    U R                  U5      $ r   r   rR   s     r!   rS   EfficientVitStage.forwardw      {{1~rP   r  )FNN)rU   rV   rW   rX   r[   r   rF   r^   r]   r\   rE   rS   r_   r`   ra   s   @r!   r   r   9  s     $;-;- ;- 	;-
 RYY;- BII;-  ;- ;- ;- ;-z rP   r   c                      ^  \ rS rSr    SS\S\S\S\\R                     S\\R                     S\S\S	\4U 4S
 jjjr	S r
SrU =r$ )EfficientVitLargeStagei{  r   r   r   r:   r;   r   r   r   c                   > XS.n[         TU ]  5         [        [        SUUSU(       a  SOSUUU=(       d    UU(       a  SOSS.UD6S 5      /nUnU(       a3  [	        U5       H#  nUR                  [        SUUSUUS	.UD65        M%     O[[	        U5       HL  nUR                  [        [        SUUS
SUUUU(       a  SOSS.UD6[        R                  " 5       5      5        MN     [        R                  " U6 U l
        g )Nr=   r+         r   r   )r3   r4   r5   rs   r:   r;   r   r   r   r   r   r   rC   r   )rL   r   r   r   r:   r;   r   r   r   r>   r?   rM   r   r    r   rN   s                  r!   rE   EfficientVitLargeStage.__init__|  s%    / 
"$#,R"%#$2
(29
 
 
  5\% $*!)%&#-"+ 	 " 5\m% 
$*%, %&#-"+#-0:9
 
 KKM  "  mmV,rP   c                 $    U R                  U5      $ r   r  rR   s     r!   rS   EfficientVitLargeStage.forward  r  rP   r  )FFNN)rU   rV   rW   rX   r[   r   rF   r^   r\   rE   rS   r_   r`   ra   s   @r!   r  r  {  s     $$>->- >- 	>-
 RYY>- BII>- >- >- >- >-@ rP   r  c                      ^  \ rS rSrSS\R
                  \R                  SSSS4S\S\\   S	\S
\	S\
\R                     S\\
\R                        S\S\	4U 4S jjjrSS	\S\\   4S jjrSS\4S jjrSrU =r$ )ClassifierHeadi    r2   avgr   Nr3   widthsnum_classesr9   r:   r;   	pool_typenorm_epsc                 *  > XS.n[         TU ]  5         X l        US   U l        U(       d   S5       e[	        XS   S4XVS.UD6U l        [        USS9U l        [        R                  " [        R                  " US   US   4S	S
0UD6[        R                  " US   4SU0UD6Ub  U" SS9O[        R                  " 5       [        R                  " US
S9US:  a  [        R                  " US   U4S	S0UD6O[        R                  " 5       5      U l        g )Nr=   r   Cannot disable poolingr   r   )r:   r;   Tr  flattenr8   Fr   r@   )rD   rE   r  rB   r/   r   r   global_poolrF   r   Linear	LayerNormrI   rG   
classifier)rL   r3   r  r  r9   r:   r;   r  r  r>   r?   rM   rN   s               r!   rE   ClassifierHead.__init__  s     /"2J222y";q	1oolno/)TR--IIfQi=="=LL77B7'0'<Id#"++-JJw.BMPQ/BIIfQi>4>2>WYWbWbWd
rP   c                     Ub  U(       d   S5       e[        USS9U l        US:  a.  [        R                  " U R                  USS9U R
                  S'   g [        R                  " 5       U R
                  S'   g )Nr  Tr  r   )r8   r   )r   r  rF   r  rB   r  rI   )rL   r  r  s      r!   resetClassifierHead.reset  sc     66693iQUWD?"$))D,=,={QU"VDOOB"$++-DOOBrP   
pre_logitsc                 &   U R                  U5      nU R                  U5      nU(       aV  U R                  S   " U5      nU R                  S   " U5      nU R                  S   " U5      nU R                  S   " U5      nU$ U R                  U5      nU$ )Nr   r   r+   r1   )r   r  r  rL   r   r"  s      r!   rS   ClassifierHead.forward  s    LLOQ"1%A"1%A"1%A"1%A  "ArP   )r  r  r   rB   r  r   F)rU   rV   rW   rX   rF   rY   r   r[   r   r]   r   r^   r   r   rE   r   r\   rS   r_   r`   ra   s   @r!   r  r    s    
  $*,..35<<""

 I
 	

 
 RYY
  RYY0
 
 
 
:0 0# 0T  rP   r  c                     ^  \ rS rSrSSSSS\R
                  \R                  SSSSS	S	4S
\S\\S4   S\\S4   S\S\	S\
\R                     S\
\R                     S\S\\S4   S\	S\4U 4S jjjr\R                  R                   S+S j5       r\R                  R                   S,S j5       r\R                  R                   S\R                  4S j5       rS-S\S\\   4S jjr     S.S\R,                  S\\\\\   4      S\S\S \S!\S\\\R,                     \\R,                  \\R,                     4   4   4S" jjr   S/S\\\\   4   S#\S$\4S% jjrS& rS+S'\4S( jjrS) rS*rU =r $ )0r   i  r1   rC   r   r   r  r2   r  Nin_chansr  .depthsr   rs   r:   r;   r  head_widths	drop_rater  c                   > XS.n[         TU ]  5         SU l        Xl        Xl        Xl        [        XS   US   Xg40 UD6U l        U R                  R                  n/ U l	        [        R                  " 5       U l        US   n[        [        USS  USS  5      5       Hb  u  nu  nnU R                  R                  [!        UU4UUUUUUS:  S.UD65        US-  nUnU =R                  [#        UUSU 3S9/-  sl	        Md     UU l        ['        U R$                  4U	UU
U R                  S	.UD6U l        U R(                  R$                  U l        g )
Nr=   Fr   r   r+   )r   r:   r;   rs   r   r   stages.num_chs	reductionmodule)r  r  r9   r  )rD   rE   grad_checkpointingr  r  r(  r   stemr5   feature_inforF   r   stages	enumeratezipr   r   dictrB   r  headhead_hidden_size)rL   r(  r  r)  r   rs   r:   r;   r  r*  r+  r  r>   r?   rM   r5   r3   r   wdrN   s                       r!   rE   EfficientVit.__init__  s}     /"'&&  !9fQiURTU	!! mmoQi"3vabz6!":#>?IAv1KK0
  %#)!q&
  
  
 aKFK${fW^_`^aUb"c!dd @  ("
#&&
 
	 !%		 6 6rP   c                 0    [        SU(       a  SOSS/S9nU$ Nz^stemz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)r3  r   r8  rL   coarsematchers      r!   group_matcherEfficientVit.group_matcher0  '    (.$455
 rP   c                     Xl         g r   r2  rL   enables     r!   set_grad_checkpointing#EfficientVit.set_grad_checkpointing;      "(rP   r)   c                 4    U R                   R                  S   $ Nr   r9  r  rL   s    r!   get_classifierEfficientVit.get_classifier?      yy##B''rP   c                 F    Xl         U R                  R                  X5        g r   r  r9  r   rL   r  r  s      r!   reset_classifierEfficientVit.reset_classifierC      &		1rP   r   indicesrJ   
stop_early
output_fmtintermediates_onlyc                    US;   d   S5       e/ n[        [        U R                  5      U5      u  pU R                  U5      n[        R
                  R                  5       (       d  U(       d  U R                  n
OU R                  SU	S-    n
[        U
5       He  u  pU R                  (       a/  [        R
                  R                  5       (       d  [        X5      nOU" U5      nX;   d  MT  UR                  U5        Mg     U(       a  U$ X4$ a  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    norm: Apply norm layer to compatible intermediates
    stop_early: Stop iterating over blocks when last desired intermediate hit
    output_fmt: Shape of intermediate feature outputs
    intermediates_only: Only return intermediate features
Returns:

)NCHWzOutput shape must be NCHW.Nr   r   r&   r5  r3  r   r   r   r6  r2  r   r   rL   r   r[  rJ   r\  r]  r^  intermediatestake_indices	max_indexr5  feat_idxstages                r!   forward_intermediates"EfficientVit.forward_intermediatesG      * Y&D(DD&"6s4;;7G"Q IIaL99!!##:[[F[[)a-0F(0OH&&uyy/E/E/G/G"6-!H'$$Q'  1   rP   
prune_norm
prune_headc                     [        [        U R                  5      U5      u  pEU R                  SUS-    U l        U(       a  U R                  SS5        U$ z?Prune layers not required for specified intermediates.
        Nr   r    r   r&   r5  rX  rL   r[  rl  rm  re  rf  s         r!   prune_intermediate_layers&EfficientVit.prune_intermediate_layersu  K     #7s4;;7G"Qkk.9q=1!!!R(rP   c                     U R                  U5      nU R                  (       a;  [        R                  R	                  5       (       d  [        U R                  U5      nU$ U R                  U5      nU$ r   r3  r2  r   r   r   r   r5  rR   s     r!   forward_featuresEfficientVit.forward_features  U    IIaL""599+A+A+C+Ct{{A.A  AArP   r"  c                 P    U(       a  U R                  XS9$ U R                  U5      $ N)r"  r9  r$  s      r!   forward_headEfficientVit.forward_head  #    6@tyyy2RdiiPQlRrP   c                 J    U R                  U5      nU R                  U5      nU$ r   rx  r~  rR   s     r!   rS   EfficientVit.forward  '    !!!$a rP   )
r4  r  r2  r9  r:  r(  r  rB   r5  r3  r&  Tr   NFFra  Fr   FT)!rU   rV   rW   rX   rF   rY   r   r[   r   r]   r   r^   r   rE   r   r   ignorerD  rK  rR  r   rX  Tensorr	   r   r\   ri  rs  rx  r~  rS   r_   r`   ra   s   @r!   r   r     s@    &(&("#*,..)+$+-"#8787 #s(O87 #s(O	87
 87  87 RYY87 BII87 87 sCx87 87 87 87t YY  YY) ) YY(		 ( (2C 2hsm 2 8<$$',, ||,  eCcN34,  	, 
 ,  ,  !%,  
tELL!5tELL7I)I#JJ	K, ` ./$#	3S	>*  	S$ S rP   c                     ^  \ rS rSrSSSS\R
                  \SSSSSS	S	4S
\S\\S4   S\\S4   S\S\	\R                     S\	\R                     S\S\\S4   S\S\S\4U 4S jjjr\R                  R                   S+S j5       r\R                  R                   S,S j5       r\R                  R                   S\R                  4S j5       rS-S\S\\   4S jjr     S.S\R,                  S\\\\\   4      S\S\S \S!\S\\\R,                     \\R,                  \\R,                     4   4   4S" jjr   S/S\\\\   4   S#\S$\4S% jjrS& rS+S'\4S( jjrS) rS*rU =r $ )0r   i  r1   rC   r   r  r2   r  gHz>Nr(  r  .r)  r   r:   r;   r  r*  r+  r  r  c                   > XS.n[         TU ]  5         SU l        Xpl        Xl        Xl        Xl        [        XPR                  S9n[        XS   US   XV4SS0UD6U l	        U R                  R                  n/ U l        [        R                  " 5       U l        US   n[        [!        USS  USS  5      5       He  u  nu  nnU R                  R#                  [%        UU4UUUUUS:  US	:  S
.UD65        US	-  nUnU =R                  ['        UUSU 3S9/-  sl        Mg     UU l        [+        U R(                  4UU
U	U R                  UU R                  S.UD6U l        U R,                  R(                  U l        g )Nr=   F)r   r   r   r   r   r1   r+   )r   r:   r;   r   r   r   r-  r.  )r  r  r9   r  r;   r  )rD   rE   r2  r  r  r(  r  r
   r   r3  r5   r4  rF   r   r5  r6  r7  r   r  r8  rB   r  r9  r:  )rL   r(  r  r)  r   r:   r;   r  r*  r+  r  r  r>   r?   rM   r5   r3   r   r;  r<  rN   s                       r!   rE   EfficientVitLarge.__init__  s     /"'&&  Z]];
 !9fQii[bifhi	!! mmoQi"3vabz6!":#>?IAv1KK5
  %#!q&6
  
  
 aKFK${fW^_`^aUb"c!dd @  ("	
#&&]]	
 	
	 !%		 6 6rP   c                 0    [        SU(       a  SOSS/S9nU$ r?  r@  rA  s      r!   rD  EfficientVitLarge.group_matcher  rF  rP   c                     Xl         g r   rH  rI  s     r!   rK  (EfficientVitLarge.set_grad_checkpointing  rM  rP   r)   c                 4    U R                   R                  S   $ rO  rP  rQ  s    r!   rR   EfficientVitLarge.get_classifier  rT  rP   c                 F    Xl         U R                  R                  X5        g r   rV  rW  s      r!   rX  "EfficientVitLarge.reset_classifier  rZ  rP   r   r[  rJ   r\  r]  r^  c                    US;   d   S5       e/ n[        [        U R                  5      U5      u  pU R                  U5      n[        R
                  R                  5       (       d  U(       d  U R                  n
OU R                  SU	S-    n
[        U
5       He  u  pU R                  (       a/  [        R
                  R                  5       (       d  [        X5      nOU" U5      nX;   d  MT  UR                  U5        Mg     U(       a  U$ X4$ r`  rb  rc  s                r!   ri  'EfficientVitLarge.forward_intermediates  rk  rP   rl  rm  c                     [        [        U R                  5      U5      u  pEU R                  SUS-    U l        U(       a  U R                  SS5        U$ ro  rq  rr  s         r!   rs  +EfficientVitLarge.prune_intermediate_layers  ru  rP   c                     U R                  U5      nU R                  (       a;  [        R                  R	                  5       (       d  [        U R                  U5      nU$ U R                  U5      nU$ r   rw  rR   s     r!   rx  "EfficientVitLarge.forward_features&  rz  rP   r"  c                 P    U(       a  U R                  XS9$ U R                  U5      $ r|  r}  r$  s      r!   r~  EfficientVitLarge.forward_head.  r  rP   c                 J    U R                  U5      nU R                  U5      nU$ r   r  rR   s     r!   rS   EfficientVitLarge.forward1  r  rP   )r4  r  r2  r9  r:  r(  r  r  rB   r5  r3  r&  r  r   r  r  )!rU   rV   rW   rX   rF   rY   r   r[   r   r   r^   r   r]   rE   r   r   r  rD  rK  rR  r   rX  r  r	   r   r\   ri  rs  rx  r~  rS   r_   r`   ra   s   @r!   r   r     s<    "$"$&(nn%- ')<7<7 c3h<7 c3h	<7
 <7 O<7 		?<7 <7 38_<7 <7 <7 <7 <7| YY  YY) ) YY(		 ( (2C 2hsm 2 8<$$',, ||,  eCcN34,  	, 
 ,  ,  !%,  
tELL!5tELL7I)I#JJ	K, ` ./$#	3S	>*  	S$ S rP   c                 0    U S[         [        SSSSSSS.
UE$ )	Nr  zstem.in_conv.convzhead.classifier.4gffffff?z
apache-2.0)r1      r  )   r  )
urlr  meanstd
first_convr  crop_pctlicense
input_size	pool_sizer   )r  kwargss     r!   _cfgr  7  s4    %#))#  rP   zefficientvit_b0.r224_in1kztimm/)	hf_hub_idzefficientvit_b1.r224_in1kzefficientvit_b1.r256_in1k)r1      r  )r   r   r   )r  r  r  r  zefficientvit_b1.r288_in1k)r1      r  )	   r  zefficientvit_b2.r224_in1kzefficientvit_b2.r256_in1kzefficientvit_b2.r288_in1kzefficientvit_b3.r224_in1kzefficientvit_b3.r256_in1kzefficientvit_b3.r288_in1kzefficientvit_l1.r224_in1k)r  r  zefficientvit_l2.r224_in1kzefficientvit_l2.r256_in1kzefficientvit_l2.r288_in1kzefficientvit_l2.r384_in1k)r1     r  )   r  zefficientvit_l3.r224_in1kzefficientvit_l3.r256_in1k)r1   @  r  )
   r  )zefficientvit_l3.r320_in1kzefficientvit_l3.r384_in1kc           	      b    UR                  SS5      n[        [        U U4S[        SUS90UD6nU$ Nout_indices)r   r   r+   r1   feature_cfgT)flatten_sequentialr  )popr   r   r8  variant
pretrainedr  r  models        r!   _create_efficientvitr    sG    **]L9K  DkJ	
 E LrP   c           	      b    UR                  SS5      n[        [        U U4S[        SUS90UD6nU$ r  )r  r   r   r8  r  s        r!   _create_efficientvit_larger    sG    **]L9K  DkJ	
 E LrP   c           	      H    [        SSSSS9n[        SSU 0[        U40 UD6D6$ )N)r   r
  r   @      )r   r+   r+   r+   r+   r
  )   i   r  r)  r   r*  r  )efficientvit_b0r8  r  r  r  
model_argss      r!   r  r    s6    #ObVbdJgjgDQ[Lf_eLfggrP   c           	      H    [        SSSSS9n[        SSU 0[        U40 UD6D6$ )N)r
  r   r  r  r  )r   r+   r1   r1   r   r
  )i   i@  r  r  )efficientvit_b1r  r  s      r!   r  r    6    %oXdfJgjgDQ[Lf_eLfggrP   c           	      H    [        SSSSS9n[        SSU 0[        U40 UD6D6$ )N)r	  0   `      r  )r   r1   r   r   r   r   i 	  i 
  r  r  )efficientvit_b2r  r  s      r!   r  r    r  rP   c           	      H    [        SSSSS9n[        SSU 0[        U40 UD6D6$ )Nr   r  r  r     )r   r   r   r   r  r   r  r  r  )efficientvit_b3r  r  s      r!   r  r    s6    &YegJgjgDQ[Lf_eLfggrP   c           	      H    [        SSSSS9n[        SSU 0[        U40 UD6D6$ )Nr  )r   r   r   r   r   r   i   i  r  r  )efficientvit_l1r8  r  r  s      r!   r  r    7    &YegJ%mJmRVWaRlekRlmmrP   c           	      H    [        SSSSS9n[        SSU 0[        U40 UD6D6$ )Nr  r   r+   r+   r   r   r   r  r  r  )efficientvit_l2r  r  s      r!   r  r    r  rP   c           	      H    [        SSSSS9n[        SSU 0[        U40 UD6D6$ )N)r  r  r  r  r  r  r   )i   i   r  r  )efficientvit_l3r  r  s      r!   r  r    s7    (2[giJ%mJmRVWaRlekRlmmrP   )r   )r   r   )Fr   NN)rp  r&  )Hr   __all__typingr   r   r   r   r	   	functoolsr
   r   torch.nnrF   torch.nn.functional
functionalr   	timm.datar   r   timm.layersr   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   	_registryr   r   r   r   anyr"   r[   r'   r,   r^   r/   rc   rp   r   r   r   r   r   r]   r   r\   r   r   r   r   r  r  r   r   r  default_cfgsr  r  r  r  r  r  r  r  r  rC   rP   r!   <module>r     s   .
/ 5 5      A E E * + 1 ' <+$$ +%%#  S  #"8sCx  S=SE#s(O  %")) %P+RYY +\.		 .b9RYY 9x/")) /dcbii cL   )		 )XBII 4 !#::: : 	:
 : : : :z+2== +\?		 ?DBRYY BJ2RYY 2j\299 \~`		 `F  % X&"X&  "	X&   FS"X&   FS"X&  "X&$   FS"%X&,   FS"-X&4  "5X&:   FS";X&B   FS"CX&J  "KX&R  "SX&Z   FS"[X&b   FS"cX&j   Hs"kX&r  "sX&z   FS"{X&B "& Hs" "& Hs"KX& Xv		 h h h h h h h h n n n n n nrP   