
    RЦi                     (   S r SSKJr  SSKJr  SSKJrJrJrJ	r	J
r
Jr  SSKrSSKJr  SSKJrJr  SSKJrJrJrJrJrJr  SS	KJr  SS
KJr  SSKJrJr  SSK J!r!J"r"  S/r# " S S\RH                  5      r% " S S\RH                  5      r& " S S\RH                  5      r' " S S\RP                  5      r) " S S\RP                  5      r* " S S\RH                  5      r+ " S S\RH                  5      r, " S S\RH                  5      r- " S S\RH                  5      r. " S  S!\RH                  5      r/ " S" S#\RH                  5      r0 " S$ S%\RH                  5      r1 " S& S\RH                  5      r2 " S' S(\25      r3S) r4\5" \5" S*S+S,S-S.9\5" S*S+S/S0S.9\5" S1S2S3S0S.9\5" S4S2S,S0S.9\5" S5S2S6S0S.9\5" S5S2S6S0S7S8S99\5" S:S;S<S0S7S8S99\5" S=S;S>S0S7S?9\5" S4S2S,S@S7S?9\5" SAS;SBS@S7S?9SC9
r6SsSD jr7StSE jr8\!" 0 SF\8" SGSH9_SI\8" SGSH9_SJ\8" SGSH9_SK\8" SGSH9_SL\8" SGSH9_SM\8" SGSNSO9_SP\8" SGSNSO9_SQ\8" SGSNSO9_SR\8" SGSNSO9_SS\8" SGSNSO9_ST\8" SUSV9_SW\8" SUSV9_SX\8" SUSV9_SY\8" SUSV9_SZ\8" SUSV9_S[\8" SUSV9_S\\8" SUSV9_\8" SUSV9\8" SUSV9\8" SUSV9S].E5      r9\"SuS^\24S_ jj5       r:\"SuS^\24S` jj5       r;\"SuS^\24Sa jj5       r<\"SuS^\24Sb jj5       r=\"SuS^\24Sc jj5       r>\"SuS^\24Sd jj5       r?\"SuS^\24Se jj5       r@\"SuS^\24Sf jj5       rA\"SuS^\24Sg jj5       rB\"SuS^\24Sh jj5       rC\"SuS^\24Si jj5       rD\"SuS^\24Sj jj5       rE\"SuS^\24Sk jj5       rF\"SuS^\24Sl jj5       rG\"SuS^\24Sm jj5       rH\"SuS^\24Sn jj5       rI\"SuS^\24So jj5       rJ\"SuS^\24Sp jj5       rK\"SuS^\24Sq jj5       rL\"SuS^\24Sr jj5       rMg)va  LeViT

Paper: `LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference`
    - https://arxiv.org/abs/2104.01136

@article{graham2021levit,
  title={LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference},
  author={Benjamin Graham and Alaaeldin El-Nouby and Hugo Touvron and Pierre Stock and Armand Joulin and Herv'e J'egou and Matthijs Douze},
  journal={arXiv preprint arXiv:22104.01136},
  year={2021}
}

Adapted from official impl at https://github.com/facebookresearch/LeViT, original copyright bellow.

This version combines both conv/linear models and fixes torchscript compatibility.

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )OrderedDict)partial)DictListOptionalTupleTypeUnionN)IMAGENET_DEFAULT_STDIMAGENET_DEFAULT_MEAN)	to_ntuple	to_2tupleget_act_layerDropPathtrunc_normal_ndgrid   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)generate_default_cfgsregister_modelLevitc                      ^  \ rS rSr        SS\S\S\S\S\S\S\S	\4U 4S
 jjjr\R                  " 5       S 5       r	S r
SrU =r$ )ConvNorm+   in_chsout_chskernel_sizestridepaddingdilationgroupsbn_weight_initc           	        > XS.n[         TU ]  5         [        R                  " XX4XVU4SS0UD6U l        [        R
                  " U40 UD6U l        [        R                  R                  U R                  R                  U5        g NdevicedtypebiasF)
super__init__nnConv2dlinearBatchNorm2dbninit	constant_weight)selfr   r   r    r!   r"   r#   r$   r%   r)   r*   dd	__class__s               P/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/levit.pyr-   ConvNorm.__init__,   sl     /iigY_rfkroqr../B/
$''...9    c           
         U R                   U R                  p!UR                  UR                  UR                  -   S-  -  nUR                  US S 2S S S 4   -  nUR
                  UR                  UR                  -  UR                  UR                  -   S-  -  -
  n[        R                  " UR                  S5      UR                  S5      UR                  SS  U R                   R                  U R                   R                  U R                   R                  U R                   R                  S9nUR                  R                  R!                  U5        UR
                  R                  R!                  U5        U$ )N      ?r   r      )r!   r"   r#   r$   )r0   r2   r5   running_varepsr+   running_meanr.   r/   sizeshaper!   r"   r#   r$   datacopy_)r6   cr2   wbms         r9   fuseConvNorm.fuse@   s   TWW2II"&&0S88HHqD$,--GGboo		1R^^bff5LQT4TTTIIFF1Iqvvay!''!"+dkk6H6HKK''$++2F2Ft{{OaOac 	
A	!r;   c                 B    U R                  U R                  U5      5      $ Nr2   r0   r6   xs     r9   forwardConvNorm.forwardM   s    wwt{{1~&&r;   rN   )r   r   r   r   r   r   NN__name__
__module____qualname____firstlineno__intfloatr-   torchno_gradrJ   rQ   __static_attributes____classcell__r8   s   @r9   r   r   +   s    
  !$%:: : 	:
 : : : : ": :( ]]_
 
' 'r;   r   c                   v   ^  \ rS rSr   S	S\S\S\4U 4S jjjr\R                  " 5       S 5       r	S r
SrU =r$ )

LinearNormQ   in_featuresout_featuresr%   c                   > XES.n[         TU ]  5         [        R                  " X4SS0UD6U l        [        R
                  " U40 UD6U l        [        R                  R                  U R                  R                  U5        g r'   )
r,   r-   r.   Linearr0   BatchNorm1dr2   r3   r4   r5   )r6   rb   rc   r%   r)   r*   r7   r8   s          r9   r-   LinearNorm.__init__R   sc     /iiLLL..44
$''...9r;   c                 6   U R                   U R                  p!UR                  UR                  UR                  -   S-  -  nUR                  US S 2S 4   -  nUR
                  UR                  UR                  -  UR                  UR                  -   S-  -  -
  n[        R                  " UR                  S5      UR                  S5      5      nUR                  R                  R                  U5        UR
                  R                  R                  U5        U$ )Nr=   r   r   )r0   r2   r5   r?   r@   r+   rA   r.   re   rB   rD   rE   )r6   lr2   rG   rH   rI   s         r9   rJ   LinearNorm.fusea   s    TWW2II"&&0S88HHqDz!GGboo		1R^^bff5LQT4TTTIIaffQi+	A	!r;   c                     U R                  U5      nU R                  UR                  SS5      5      R                  U5      $ )Nr   r   )r0   r2   flatten
reshape_asrO   s     r9   rQ   LinearNorm.forwardl   s3    KKNwwqyyA'22155r;   rN   )r   NNrS   r^   s   @r9   r`   r`   Q   sX    
 %&:: : "	: : ]]_ 6 6r;   r`   c                      ^  \ rS rSr     SS\S\S\S\S\4
U 4S jjjr\R                  " 5       S 5       r
S	 rS
rU =r$ )
NormLinearq   rb   rc   r+   stddropc                   > XgS.n[         T	U ]  5         [        R                  " U40 UD6U l        [        R
                  " U5      U l        [        R                  " X4SU0UD6U l        [        U R                  R                  US9  U R                  R                  b5  [        R                  R                  U R                  R                  S5        g g )Nr(   r+   )rr   r   )r,   r-   r.   rf   r2   Dropoutrs   re   r0   r   r5   r+   r3   r4   )
r6   rb   rc   r+   rr   rs   r)   r*   r7   r8   s
            r9   r-   NormLinear.__init__r   s     /..33JJt$	iiKKKdkk((c2;;'GGdkk..2 (r;   c                 6   U R                   U R                  p!UR                  UR                  UR                  -   S-  -  nUR
                  U R                   R                  U R                   R                  -  UR                  UR                  -   S-  -  -
  nUR                  US S S 24   -  nUR
                  c#  X@R                  R                  R                  -  nO<UR                  US S 2S 4   -  R                  S5      U R                  R
                  -   n[        R                  " UR                  S5      UR                  S5      5      nUR                  R                  R                  U5        UR
                  R                  R                  U5        U$ )Nr=   r   r   )r2   r0   r5   r?   r@   r+   rA   Tviewr.   re   rB   rD   rE   )r6   r2   ri   rG   rH   rI   s         r9   rJ   NormLinear.fuse   s)   AII"&&0S88GGdgg**TWW^^;r~~PRPVPV?V[^>^^^HHqqz!66>KK&&(((AAagJ&,,R04;;3C3CCAIIaffQi+	A	!r;   c                 `    U R                  U R                  U R                  U5      5      5      $ rM   )r0   rs   r2   rO   s     r9   rQ   NormLinear.forward   s"    {{499TWWQZ011r;   )r2   rs   r0   )Tg{Gz?        NN)rT   rU   rV   rW   rX   boolrY   r-   rZ   r[   rJ   rQ   r\   r]   r^   s   @r9   rp   rp   q   sr    
 33 3 	3
 3 3 3( ]]_ 2 2r;   rp   c                   Z   ^  \ rS rSr  SS\S\S\\R                     4U 4S jjjrSr	U =r
$ )Stem8   r   r   	act_layerc           	      p  > XES.n[         TU ]  5         SU l        U R                  S[	        XS-  S4SSS.UD65        U R                  S	U" 5       5        U R                  S
[	        US-  US-  S4SSS.UD65        U R                  SU" 5       5        U R                  S[	        US-  US4SSS.UD65        g )Nr(      conv1      r>   r   r!   r"   act1conv2act2conv3r,   r-   r!   
add_moduler   r6   r   r   r   r)   r*   r7   r8   s          r9   r-   Stem8.__init__   s     /&Q,!]!UV!]Z\!]^	,'Q,1a!cPQ[\!c`b!cd	,'Q,!^1VW!^[]!^_r;   r!   NNrT   rU   rV   rW   rX   r	   r.   Moduler-   r\   r]   r^   s   @r9   r   r      s>     `` ` BII	` `r;   r   c                   Z   ^  \ rS rSr  SS\S\S\\R                     4U 4S jjjrSr	U =r
$ )Stem16   r   r   r   c           	        > XES.n[         TU ]  5         SU l        U R                  S[	        XS-  S4SSS.UD65        U R                  S	U" 5       5        U R                  S
[	        US-  US-  S4SSS.UD65        U R                  SU" 5       5        U R                  S[	        US-  US-  S4SSS.UD65        U R                  SU" 5       5        U R                  S[	        US-  US4SSS.UD65        g )Nr(      r   r   r   r>   r   r   r   r   r   r   r   act3conv4r   r   s          r9   r-   Stem16.__init__   s     /&Q,!]!UV!]Z\!]^	,'Q,1a!cPQ[\!c`b!cd	,'Q,1a!cPQ[\!c`b!cd	,'Q,!^1VW!^[]!^_r;   r   r   r   r^   s   @r9   r   r      s>     `` ` BII	` `r;   r   c            	       \   ^  \ rS rSr   SS\S\\\\\4   4   S\4U 4S jjjrS r	Sr
U =r$ )	
Downsample   r!   
resolutionuse_poolc                    > [         TU ]  5         Xl        [        U5      U l        U(       a  [
        R                  " SUSSS9U l        g S U l        g )Nr   r   F)r!   r"   count_include_pad)r,   r-   r!   r   r   r.   	AvgPool2dpool)r6   r!   r   r   r)   r*   r8   s         r9   r-   Downsample.__init__   sA     	#J/ZbBLL61PUV	hl	r;   c                 n   UR                   u  p#nUR                  X R                  S   U R                  S   U5      nU R                  b6  U R                  UR	                  SSSS5      5      R	                  SSSS5      nO$US S 2S S U R
                  2S S U R
                  24   nUR                  USU5      $ )Nr   r   r   r>   rx   )rC   rz   r   r   permuter!   reshape)r6   rP   BNCs        r9   rQ   Downsample.forward   s    ''aFF1ooa($//!*<a@99 		!))Aq!Q/088Aq!DA!]t{{]MdkkM12AyyB""r;   )r   r   r!   )FNN)rT   rU   rV   rW   rX   r
   r   r   r-   rQ   r\   r]   r^   s   @r9   r   r      sU    
 #mm c5c?23m 	m m# #r;   r   c                   f  ^  \ rS rSr% \\\R                  4   \S'   SSSS\	R                  SS4S\S	\S
\S\S\\\\\4   4   S\S\\	R"                     4U 4S jjjr\R&                  " 5       SU 4S jj5       rSS jrSS jrSS jrSS jrS\R2                  S\R                  4S jrS rSrU =r$ )	Attention   attention_bias_cacher         @   FNdimkey_dim	num_heads
attn_ratior   use_convr   c
                   > XS.n
[         TU ]  5         U(       a  [        O[        n[	        U5      nX`l        X0l        US-  U l        X l        X#-  U l	        [        XB-  5      U l        [        XB-  5      U-  U l        XPl        U" XR                  U R                  S-  -   40 U
D6U l        [        R                   " [#        SU" 5       4SU" U R                  U4SS0U
D64/5      5      U l        US   US   -  n[        R&                  " [(        R*                  " X<40 U
D65      U l        U R/                  S	[(        R*                  " X4U[(        R0                  S9S
S9  0 U l        U R5                  5         g )Nr(         r>   actlnr%   r   r   attention_bias_idxsF
persistent)r,   r-   r   r`   r   r   r   scaler   key_attn_dimrX   val_dimval_attn_dimr   qkvr.   
Sequentialr   proj	ParameterrZ   emptyattention_biasesregister_bufferlongr   reset_parameters)r6   r   r   r   r   r   r   r   r)   r*   r7   ln_layerr   r8   s                r9   r-   Attention.__init__   s]    /'8Zz*
 "_
#/:/0
 45	A$C!2!2T5F5F5J!JQbQMM+IK 8D--sK1KKL/
 # 	
 qMJqM) "U[[-L-L M!5;;vfEJJ#Wdi 	 	k$&! 	r;   c                 f   > [         TU ]  U5        U(       a  U R                  (       a  0 U l        g g g rM   r,   trainr   r6   moder8   s     r9   r   Attention.train  )    dD--(*D% .4r;   returnc                 v    [         R                  R                  U R                  5        U R	                  5         gz"Initialize parameters and buffers.Nr.   r3   zeros_r   _init_buffersr6   s    r9   r   Attention.reset_parameters  $    
t,,-r;   c                    [         R                  " [        [         R                  " U R                  S   U[         R
                  S9[         R                  " U R                  S   U[         R
                  S95      5      R                  S5      nUSSS2S4   USSSS24   -
  R                  5       nUS   U R                  S   -  US   -   nU$ )5Compute relative position indices for attention bias.r   r(   r   .N)rZ   stackr   aranger   r   rl   abs)r6   r)   posrel_poss       r9   _compute_attention_bias_idxs&Attention._compute_attention_bias_idxs  s    kk&LL+F%**MLL+F%**M
  71: 	 sAt|$s3a<'88==?1: 22gaj@r;   c                     U R                   R                  U R                  U R                   R                  S95        0 U l        gz.Compute and fill non-persistent buffer values.)r)   Nr   rE   r   r)   r   r   s    r9   r   Attention._init_buffers  =      &&--T5M5M5T5T-U	
 %'!r;   c                 $    U R                  5         gz"Initialize non-persistent buffers.Nr   r   s    r9   init_non_persistent_buffers%Attention.init_non_persistent_buffers#      r;   r)   c                 J   [         R                  R                  5       (       d  U R                  (       a  U R                  S S 2U R
                  4   $ [        U5      nX R                  ;  a*  U R                  S S 2U R
                  4   U R                  U'   U R                  U   $ rM   rZ   jit
is_tracingtrainingr   r   strr   r6   r)   
device_keys      r9   get_attention_biasesAttention.get_attention_biases'      99!!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88r;   c                 6   U R                   (       a  UR                  u  p#pEU R                  U5      R                  X R                  SXE-  5      R                  U R                  U R                  U R                  /SS9u  pgnUR                  SS5      U-  U R                  -  U R                  UR                  5      -   n	U	R                  SS9n	XR                  SS5      -  R                  USXE5      nGOUR                  u  p*nU R                  U5      R                  X*U R                  S5      R                  U R                  U R                  U R                  /SS9u  pgnUR                  SSSS5      nUR                  SSSS5      nUR                  SSSS5      nXg-  U R                  -  U R                  UR                  5      -   n	U	R                  SS9n	X-  R                  SS5      R                  X*U R                  5      nU R!                  U5      nU$ )Nrx   r>   r   r   r   r   )r   rC   r   rz   r   splitr   r   	transposer   r   r)   softmaxr   r   r   r   )r6   rP   r   r   HWqkvattnr   s              r9   rQ   Attention.forward0  s   ==JA!hhqk&&>>2qu..3eT\\4<<QUQ]Q]4^dee.f A! KKB'!+tzz9D<U<UVWV^V^<__D<<B<'D^^B++11!R>AggGA!hhqk&&dnnb**/%t||T\\0Z`a%*b A!		!Q1%A		!Q1%A		!Q1%A54::%(A(A!(((KKD<<B<'D$$Q*2219J9JKAIIaLr;   )r   r   r   r   r   r   r   r   r   r   r   r   Tr   NrM   rT   rU   rV   rW   r   r   rZ   Tensor__annotations__r.   SiLUrX   rY   r
   r   r   r	   r   r-   r[   r   r   r   r   r   r)   r   rQ   r\   r]   r^   s   @r9   r   r      s    sELL011  "68")+' '  '  	' 
 '  c5c?23'  '  BII'  ' R ]]_+ +

'95<< 9ELL 9 r;   r   c                   v  ^  \ rS rSr% \\\R                  4   \S'   SSSSSS\	R                  SS4	S	\S
\S\S\S\S\S\\\\\4   4   S\S\S\\	R"                     4U 4S jjjr\R&                  " 5       SU 4S jj5       rSS jrS S jrSS jrSS jrS\R2                  S\R                  4S jrS rSrU =r$ )!AttentionDownsampleiJ  r   r          @r>   r   FNin_dimout_dimr   r   r   r!   r   r   r   r   c                 >  > XS.n[         TU ]  5         [        U5      nX`l        Xpl        X@l        X0l        X4-  U l        [        XS-  5      U l	        U R                  U R
                  -  U l
        US-  U l        Xl        U R                  (       a3  [        n[        [        R                   U	(       a  SOSU	(       a  SOSSS9nO["        n[        [$        4XyS.UD6nU" XR                  U R                  -   40 UD6U l        [        R(                  " [+        S	U" US
94SU" XR                  40 UD64/5      5      U l        [        R(                  " [+        SU
" 5       4SU" U R                  U40 UD64/5      5      U l        US   US   -  nUS   * U-  * US   * U-  * -  n[        R0                  " [2        R4                  " UU40 UD65      U l        U R9                  S[2        R4                  " UU4U[2        R:                  S9SS9  0 U l        U R?                  5         g )Nr(   r   r   r   r   F)r    r"   r   )r   r   downr   r   r   r   r   ) r,   r-   r   r!   r   r   r   r   rX   r   r   r   r   r   r   r.   r   r`   r   kvr   r   r  r   r   rZ   r   r   r   r   r   r   )r6   r  r  r   r   r   r!   r   r   r   r   r)   r*   r7   r   	sub_layerN_kN_qr8   s                     r9   r-   AttentionDownsample.__init__M  s    /z*
$"#/:/0 LL4>>9_
 ==H!)Aqx!QbgiI "H
[z[XZ[I6#4#4t7H7H#HOBO{Yf-.8F$5$5<<=,
    MM+IK 8D--w="=>/
 # 	
 mjm+A&()z!}n.F,GG "U[[C-N2-N O2EKKc
SYafakak4ly~$&! 	r;   c                 f   > [         TU ]  U5        U(       a  U R                  (       a  0 U l        g g g rM   r   r   s     r9   r   AttentionDownsample.train  r   r;   r   c                 v    [         R                  R                  U R                  5        U R	                  5         gr   r   r   s    r9   r   $AttentionDownsample.reset_parameters  r   r;   c                    [         R                  " [        [         R                  " U R                  S   U[         R
                  S9[         R                  " U R                  S   U[         R
                  S95      5      R                  S5      n[         R                  " [        [         R                  " SU R                  S   U R                  U[         R
                  S9[         R                  " SU R                  S   U R                  U[         R
                  S95      5      R                  S5      nUSSS2S4   USSSS24   -
  R                  5       nUS   U R                  S   -  US   -   nU$ )r   r   r(   r   )stepr)   r*   .N)	rZ   r   r   r   r   r   rl   r!   r   )r6   r)   k_posq_posr   s        r9   r   0AttentionDownsample._compute_attention_bias_idxs  s$   FLL+F%**MLL+F%**M
  71: 	 FLLDOOA.T[[W\WaWabLLDOOA.T[[W\WaWab
  71: 	 a&sD!|)<<AAC1: 22gaj@r;   c                     U R                   R                  U R                  U R                   R                  S95        0 U l        gr   r   r   s    r9   r   !AttentionDownsample._init_buffers  r   r;   c                 $    U R                  5         gr   r   r   s    r9   r   /AttentionDownsample.init_non_persistent_buffers  r   r;   r)   c                 J   [         R                  R                  5       (       d  U R                  (       a  U R                  S S 2U R
                  4   $ [        U5      nX R                  ;  a*  U R                  S S 2U R
                  4   U R                  U'   U R                  U   $ rM   r   r   s      r9   r   (AttentionDownsample.get_attention_biases  r   r;   c                 D   U R                   (       Ga?  UR                  u  p#pEUS-
  U R                  -  S-   US-
  U R                  -  S-   pvU R                  U5      R	                  X R
                  SXE-  5      R                  U R                  U R                  /SS9u  pU R                  U5      R	                  X R
                  U R                  S5      n
U
R                  SS5      U-  U R                  -  U R                  UR                  5      -   nUR                  SS9nXR                  SS5      -  R                  X R                   Xg5      nGO=UR                  u  p,nU R                  U5      R	                  X,U R
                  S5      R                  U R                  U R                  /SS9u  pUR#                  SSSS5      nU	R#                  SSSS5      n	U R                  U5      R	                  USU R
                  U R                  5      R#                  SSSS5      n
X-  U R                  -  U R                  UR                  5      -   nUR                  SS9nX-  R                  SS5      R                  USU R                   5      nU R%                  U5      nU$ )Nr   rx   r>   r   r  r   r   )r   rC   r!   r  rz   r   r  r   r   r  r  r   r   r)   r  r   r   r   r   )r6   rP   r   r   r  r  HHWWr  r	  r  r
  r   s                r9   rQ   AttentionDownsample.forward  s/   ===JA!!e+a/!a%DKK1G!1K771:??1nnb!%@FFVZVbVbGcijFkDAq	q..$,,CAKKB'!+tzz9D<U<UVWV^V^<__D<<B<'D^^B++44Q8I8I2RAggGA!771:??1<BBDLLRVR^R^C_efBgDA		!Q1%A		!Q1%Aq	q"dnndllCKKAqRSUVWA54::%(A(A!(((KKD<<B<'D$$Q*221b$:K:KLAIIaLr;   )r   r   r   r   r  r   r   r  r   r   r!   r   r   r   r  r  rM   r  r^   s   @r9   r  r  J  s   sELL011  #68"")+7 7  7  	7 
 7  7  7  c5c?237  7  7  BII7  7 r ]]_+ +

'95<< 9ELL 9 r;   r  c                      ^  \ rS rSrSrSSS\R                  SSS4S\S\\   S\\   S	\	S
\
\R                     S\4U 4S jjjrS rSrU =r$ )LevitMlpi  zKMLP for Levit w/ normalization + ability to switch btw conv and linear
    NFr~   rb   hidden_featuresrc   r   r   rs   c	                   > XxS.n	[         TU ]  5         U=(       d    UnU=(       d    UnU(       a  [        O[        n
U
" X40 U	D6U l        U" 5       U l        [        R                  " U5      U l        U
" X#4SS0U	D6U l	        g )Nr(   r%   r   )
r,   r-   r   r`   ln1r   r.   ru   rs   ln2)r6   rb   r3  rc   r   r   rs   r)   r*   r7   r   r8   s              r9   r-   LevitMlp.__init__  su     /#2{)8['8ZK?B?;JJt$	OR!RrRr;   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ rM   )r5  r   rs   r6  rO   s     r9   rQ   LevitMlp.forward  s;    HHQKHHQKIIaLHHQKr;   )r   rs   r5  r6  )rT   rU   rV   rW   __doc__r.   r  rX   r   r   r	   r   rY   r-   rQ   r\   r]   r^   s   @r9   r2  r2    s    
 .2*.")+SS &c]S #3-	S
 S BIIS S S, r;   r2  c                      ^  \ rS rSrSSS\R
                  SSSSSSS4S	\S
\S\S\S\S\S\\R                     S\
\\R                        S\\\\\4   4   S\S\S\4U 4S jjjrS rSrU =r$ )LevitDownsamplei  r   r   r  Nr   Fr~   r  r  r   r   r   	mlp_ratior   attn_act_layerr   r   r   	drop_pathc                   > XS.n[         TU ]  5         U=(       d    Un[        SUUUUUUU	U
US.	UD6U l        [	        U[        X&-  5      4U
US.UD6U l        US:  a  [        U5      U l
        g [        R                  " 5       U l
        g )Nr(   )	r  r  r   r   r   r   r   r   r   r   r   r~    )r,   r-   r  attn_downsampler2  rX   mlpr   r.   Identityr?  )r6   r  r  r   r   r   r=  r   r>  r   r   r   r?  r)   r*   r7   r8   s                   r9   r-   LevitDownsample.__init__  s    " /'492  
!$! 
  
 #$
 	

 
 1:B),BKKMr;   c                 l    U R                  U5      nXR                  U R                  U5      5      -   nU$ rM   rC  r?  rD  rO   s     r9   rQ   LevitDownsample.forward  s0      #txx{++r;   rH  )rT   rU   rV   rW   r.   r  rX   rY   r	   r   r   r
   r   r   r-   rQ   r\   r]   r^   s   @r9   r<  r<    s      "!)+8<68""!)R)R )R 	)R
 )R )R )R BII)R %T"))_5)R c5c?23)R )R )R )R )RV r;   r<  c                      ^  \ rS rSrSSSSS\R
                  SSSS4
S	\S
\S\S\S\S\\\	\\4   4   S\
S\\R                     S\\\R                        S\4U 4S jjjrS rSrU =r$ )
LevitBlocki   r   r   r  r   FNr~   r   r   r   r   r=  r   r   r   r>  r?  c                 h  > XS.n[         TU ]  5         U	=(       d    Un	[        SUUUUUUU	S.UD6U l        U
S:  a  [	        U
5      O[
        R                  " 5       U l        [        U[        X-  5      4UUS.UD6U l
        U
S:  a  [	        U
5      U l        g [
        R                  " 5       U l        g )Nr(   )r   r   r   r   r   r   r   r~   rA  rB  )r,   r-   r   r
  r   r.   rE  
drop_path1r2  rX   rD  
drop_path2)r6   r   r   r   r   r=  r   r   r   r>  r?  r)   r*   r7   r8   s                 r9   r-   LevitBlock.__init__!  s     /'49 	!!$	 		 2;R(9-R[[] 
 	

 
 2;R(9-R[[]r;   c                     XR                  U R                  U5      5      -   nXR                  U R                  U5      5      -   nU$ rM   )rM  r
  rN  rD  rO   s     r9   rQ   LevitBlock.forwardI  s9    		!--,,r;   )r
  rM  rN  rD  )rT   rU   rV   rW   r.   r  rX   rY   r
   r   r   r	   r   r   r-   rQ   r\   r]   r^   s   @r9   rK  rK     s    
  "!68")+8<!&S&S &S 	&S
 &S &S c5c?23&S &S BII&S %T"))_5&S &S &SP r;   rK  c                      ^  \ rS rSrSSSS\R
                  SSSSS	SS4S
\S\S\S\S\S\S\S\\R                     S\
\\R                        S\\\\\4   4   S\S\S\4U 4S jjjrS rSrU =r$ )
LevitStageiO  r   r   r   Nr    Fr~   r  r  r   depthr   r   r=  r   r>  r   
downsampler   r?  c                   > XS.n[         TU ]  5         [        U
5      n
U(       a:  [        UU4UX-  SSUU	U
UUS.	UD6U l        U
 Vs/ s H  nUS-
  S-  S-   PM     n
nO!X:X  d   e[
        R                  " 5       U l        / n[        U5       H  nU[        UU4UUUUU	U
UUS.UD6/-  nM     [
        R                  " U6 U l
        g s  snf )Nr(   r   r  )	r   r   r   r=  r   r>  r   r   r?  r   r>   )r   r   r=  r   r>  r   r   r?  )r,   r-   r   r<  rV  r.   rE  rangerK  r   blocks)r6   r  r  r   rU  r   r   r=  r   r>  r   rV  r   r?  r)   r*   r7   rrY  _r8   s                       r9   r-   LevitStage.__init__P  s   $ /z*
-   +#-%!# DO 5??Jq1q5Q,*JJ?J$$$ kkmDOuAz $%##-%!#   F  mmV,) @s   Cc                 J    U R                  U5      nU R                  U5      nU$ rM   )rV  rY  rO   s     r9   rQ   LevitStage.forward  s"    OOAKKNr;   )rY  rV  )rT   rU   rV   rW   r.   r  rX   rY   r	   r   r   r
   r   r   r   r-   rQ   r\   r]   r^   s   @r9   rS  rS  O  s      #")+8<68 "!!9-9- 9- 	9-
 9- 9- 9- 9- BII9- %T"))_59- c5c?239- 9- 9- 9- 9-v r;   rS  c            '         ^  \ rS rSrSr                     S2S\\\\\4   4   S\S\S\\S4   S	\S
\\S4   S\\\\S4   4   S\\\\S4   4   S\\\\S4   4   S\	\
R                     S\	\   S\S\S\S\	\   S\S\S\S\4&U 4S jjjrS3S\4S jjrS3S\
R                  S\SS4S jjr\R$                  R&                  S 5       r\R$                  R&                  S4S j5       r\R$                  R&                  S3S  j5       r\R$                  R&                  S\
R                  4S! j5       rS5S\S\	\   4S" jjr     S6S#\R2                  S$\	\\\\   4      S%\S&\S'\S(\S\\\R2                     \\R2                  \\R2                     4   4   4S) jjr   S7S$\\\\   4   S*\S+\4S, jjrS- rS4S.\4S/ jjrS0 rS1r U =r!$ )8r   i  zVision Transformer with support for patch or hybrid CNN input stage

NOTE: distillation is defaulted to True since pretrained weights use it, will cause problems
w/ train scripts that don't take tuple outputs,
Nimg_sizein_chansnum_classes	embed_dim.r   rU  r   r   r=  stem_backbonestem_stride	stem_typedown_opr   r>  r   global_pool	drop_ratedrop_path_ratec                   > [         T U ]  5         UUS.n[        U5      n[        U=(       d    U5      nUU l        X0l        X l        UU l        US   =U l        U l        X@l	        UU l
        SU l        / U l        [        U5      n[        U5      U:X  d   e[        U5      " U5      n[        U5      " U5      n[        U5      " U	5      n	U
b  US:  d   eXl        UnOQUS;   d   eUS:X  a  [!        X$S   4SU0UD6U l        O[#        X$S   4SU0UD6U l        U R                  R$                  n['        [)        [+        U5      [+        U5      5       VVs/ s H  u  nnUU-  PM     snn5      nUS   n/ n[-        U5       H  nUS:  a  SOS	nU[/        UUU   U4UU   UU   UU   U	U   UUUUUS:X  a  UOS
US.
UD6/-  nUU-  n['        U Vs/ s H  nUS	-
  U-  S	-   PM     sn5      nU =R                  [1        UU   USU 3S9/-  sl        UU   nM     [2        R4                  " U6 U l        US:  a  [9        US   U4SU0UD6O[2        R:                  " 5       U l        U R?                  SS9  g s  snnf s  snf )Nr(   rx   Fr>   )s16s8rl  r   r   r   rT  )
rU  r   r   r=  r   r>  r   r   rV  r?  zstages.)num_chs	reductionmodulers   needs_reset) r,   r-   r   r   rb  ra  rh  num_featureshead_hidden_sizerc  ri  grad_checkpointingfeature_infolenr   stemr   r   r!   tuplezipr   rX  rS  dictr.   r   stagesrp   rE  headinit_weights)!r6   r`  ra  rb  rc  r   rU  r   r   r=  rd  re  rf  rg  r   r>  r   rh  ri  rj  r)   r*   r7   
num_stagesr!   ipr   r  r|  stage_striderZ  r8   s!                                   r9   r-   Levit.__init__  s   0 	/!),	&~'BC & &4=bMAD1"""'^
5zZ'''j))4	z*:6
j))4	$!###%I F---E!"8q\UYURTU	!(aLTITQST	YY%%Fs9X3F	RXHY/Z[/Ztq!AF/Z[\
1z"A !A11Lz! Ah#A,%a=#A,#-%!&2a&77R(   F  l"FZPZQ< 7! ;ZPQJ$y|vX_`a_bVc"d!eeq\F+ #, mmV, U`bcTcJy}kP	PRPikititiv	 	e,A \.  Qs   I3
I9
rr  c                 J    U R                  [        U R                  US95        g )Nrq  )applyr   _init_weights)r6   rr  s     r9   r~  Levit.init_weights  s    

74--;GHr;   rI   r   c                 X    U(       a#  [        US5      (       a  UR                  5         g g g )Nr   )hasattrr   )r6   rI   rr  s      r9   r  Levit._init_weights  s%    71&899  :;r;   c                 z    U R                  5       R                  5        Vs1 s H  nSU;   d  M  UiM     sn$ s  snf )Nr   )
state_dictkeysrO   s     r9   no_weight_decayLevit.no_weight_decay  s4    ??,113O3a7IQ7N3OOOs   
88c                     [        SSS/S9nU$ )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )rx  rY  )r{  )r6   coarsematchers      r9   group_matcherLevit.group_matcher  s    4-/CD
 r;   c                     Xl         g rM   )ru  r6   enables     r9   set_grad_checkpointingLevit.set_grad_checkpointing      "(r;   c                     U R                   $ rM   )r}  r   s    r9   get_classifierLevit.get_classifier  s    yyr;   c                     Xl         Ub  X l        US:  a$  [        U R                  XR                  S9U l        g [
        R                  " 5       U l        g Nr   )rs   )rb  rh  rp   rs  ri  r.   rE  r}  r6   rb  rh  s      r9   reset_classifierLevit.reset_classifier  sM    &"*DORSO {A	Y[YdYdYf 		r;   rP   indicesnorm
stop_early
output_fmtintermediates_onlyc           	      P   US;   d   S5       e/ n[        [        U R                  5      U5      u  pU R                  U5      nUR                  u  ppU R
                  (       d!  UR                  S5      R                  SS5      n[        R                  R                  5       (       d  U(       d  U R                  nOU R                  SU	S-    n[        U5       H  u  nnU R                  (       a0  [        R                  R                  5       (       d  [        UU5      nOU" U5      nX;   aW  U R
                  (       a  UR                  U5        O4UR                  UR                  XUS5      R!                  SSSS5      5        US-   S-
  S-  nUS-   S-
  S-  nM     U(       a  U$ X4$ )	a  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    norm: Apply norm layer to compatible intermediates
    stop_early: Stop iterating over blocks when last desired intermediate hit
    output_fmt: Shape of intermediate feature outputs
    intermediates_only: Only return intermediate features
Returns:

)NCHWzOutput shape must be NCHW.r>   r   Nrx   r   r   )r   rw  r|  rx  rC   r   rl   r  rZ   r   is_scripting	enumerateru  r   appendr   r   )r6   rP   r  r  r  r  r  intermediatestake_indices	max_indexr   r   r  r  r|  feat_idxstages                    r9   forward_intermediatesLevit.forward_intermediates  sg   * Y&D(DD&"6s4;;7G"Q IIaLWW
a}}		!&&q!,A99!!##:[[F[[)a-0F(0OHe&&uyy/E/E/G/Gua(!H'==!((+!((1B)?)G)G1aQR)STQq AQq A  1   r;   
prune_norm
prune_headc                     [        [        U R                  5      U5      u  pEU R                  SUS-    U l        U(       a  U R                  SS5        U$ )z?Prune layers not required for specified intermediates.
        Nr   r   rT  )r   rw  r|  r  )r6   r  r  r  r  r  s         r9   prune_intermediate_layersLevit.prune_intermediate_layersH  sK     #7s4;;7G"Qkk.9q=1!!!R(r;   c                 F   U R                  U5      nU R                  (       d!  UR                  S5      R                  SS5      nU R                  (       a;  [
        R                  R                  5       (       d  [        U R                  U5      nU$ U R                  U5      nU$ )Nr>   r   )
rx  r   rl   r  ru  rZ   r   r  r   r|  rO   s     r9   forward_featuresLevit.forward_featuresV  sv    IIaL}}		!&&q!,A""599+A+A+C+Ct{{A.A  AAr;   
pre_logitsc                     U R                   S:X  a/  U R                  (       a  UR                  SS9OUR                  SS9nU(       a  U$ U R                  U5      $ )Navgr  rx   r   r   )rh  r   meanr}  )r6   rP   r  s      r9   forward_headLevit.forward_head`  sG    u$(,8$166a6=Aq0DIIaL0r;   c                 J    U R                  U5      nU R                  U5      nU$ rM   )r  r  rO   s     r9   rQ   Levit.forwarde  s'    !!!$a r;   )ri  rc  rv  rh  ru  r}  rt  ra  rb  rs  r|  rx  r   )   r     )   @   )   )r   r  r  NNrl  	subsample
hard_swishNFr  r~   r~   NNr  FrM   )NFFr  F)r   FT)"rT   rU   rV   rW   r:  r
   rX   r   rY   r   r.   r   r   r   r-   r~  r  rZ   r   ignorer  r  r  r  r  r  r   r  r  r  r  rQ   r\   r]   r^   s   @r9   r   r     sJ    58#)/%*59:<9;15)-"&),0"$!$&-W-CsCx01W- W- 	W-
 S#XW- W- c?W- S%S/12W- eU5#:%667W- UE%*$556W- $BII.W- "#W- W- W- W-  %SM!W-" #W-$ %W-& 'W-( ")W- W-rI I!ryy !t !t ! YYP P YY  YY) ) YY		  gC gx} g 8<$$',3 ||3  eCcN343  	3 
 3  3  !%3  
tELL!5tELL7I)I#JJ	K3 n ./$#	3S	>*  	1$ 1
 r;   c                      ^  \ rS rSrU 4S jr\R                  R                  S\R                  4S j5       r
SS\S\\   4S jjr\R                  R                  SS j5       rSS	\4S
 jjrSrU =r$ )LevitDistilledik  c                   > [         TU ]  " U0 UD6  UR                  SS 5      UR                  SS 5      S.nU R                  S:  a!  [	        U R
                  U R                  40 UD6O[        R                  " 5       U l        SU l	        g )Nr)   r*   r(   r   F)
r,   r-   getrb  rp   rs  r.   rE  	head_distdistilled_training)r6   argskwargsr7   r8   s       r9   r-   LevitDistilled.__init__l  sw    $)&)

8T2VZZQU=VWRVRbRbefRfD$5$5t7G7GN2Nlnlwlwly"'r;   r   c                 2    U R                   U R                  4$ rM   )r}  r  r   s    r9   r  LevitDistilled.get_classifierr  s    yy$..((r;   rb  rh  c                    Xl         Ub  X l        US:  a  [        U R                  XR                  S9O[
        R                  " 5       U l        US:  a  [        U R                  U5      U l        g [
        R                  " 5       U l        g r  )	rb  rh  rp   rs  ri  r.   rE  r}  r  r  s      r9   r  LevitDistilled.reset_classifierv  st    &"*DORSO {AY[YdYdYf 		GRUVD$5$5{C\^\g\g\ir;   c                     Xl         g rM   )r  r  s     r9   set_distilled_training%LevitDistilled.set_distilled_training~  r  r;   r  c                 r   U R                   S:X  a/  U R                  (       a  UR                  SS9OUR                  SS9nU(       a  U$ U R                  U5      U R	                  U5      p1U R
                  (       a7  U R                  (       a&  [        R                  R                  5       (       d  X4$ X-   S-  $ )Nr  r  r   r   r>   )
rh  r   r  r}  r  r  r   rZ   r   r  )r6   rP   r  x_dists       r9   r  LevitDistilled.forward_head  s    u$(,8$166a6=AHIIaL$.."36""t}}UYY=S=S=U=U9 J!##r;   )r  rh  r}  r  rb  rM   r  r  )rT   rU   rV   rW   r-   rZ   r   r  r.   r   r  rX   r   r   r  r  r   r  r\   r]   r^   s   @r9   r  r  k  sz    ( YY)		 ) )jC jhsm j YY) )$$ $ $r;   r  c                     SU ;   a  U S   n U R                  5        VVs0 s H  u  p#SU;  d  M  X#_M     n nnU $ s  snnf )Nmodelr   )items)r  r  r  r	  s       r9   checkpoint_filter_fnr    sR    *(
 $.#3#3#5X#5419NVW9W$!$#5JX  Ys   ::)        r   )r      r   )r>   r   r   )rc  r   r   rU  )r   r   r  )r   r   r   )r  i   r      )r      r  )r  r     )r  r     )r  	   r  silurm  )rc  r   r   rU  r   rf  )r    i  r  )r   
   r   )r  r  i   )r   r  r   )rc  r   r   rU  r   )r   r   r  )r  r  r  )r   r  r  )

levit_128s	levit_128	levit_192	levit_256	levit_384levit_384_s8levit_512_s8	levit_512
levit_256d
levit_512dc           	      h   SU ;   nUR                  SS5      nUR                  SS5      (       a  U(       d  UR                  SS5        Uc&  U [        ;   a  U nOU(       a  U R	                  SS5      n[        [        U   40 UD6n[        U(       a  [        O[        U U4[        [        S	US
9S.UD6nU$ )N_convout_indices)r   r   r>   features_onlyFfeature_clsgetterrT  T)flatten_sequentialr  )pretrained_filter_fnfeature_cfg)
popr  
setdefault
model_cfgsreplacer{  r   r  r   r  )	variantcfg_variant
pretrained	distilledr  is_convr  	model_cfgr  s	            r9   create_levitr    s     G**]I6Kzz/5))'-2j !K!//'26KZ,77I # 2DkJ E Lr;   c                 4    U SSS SSS[         [        SSSS	.UE$ )
Nr  )r   r  r  g?bicubicTzstem.conv1.linear)head.linearzhead_dist.linearz
apache-2.0)urlrb  
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer  rr   
first_conv
classifierlicense)r   r   )r  r  s     r9   _cfgr    s6    =t%.B)9\  r;   zlevit_128s.fb_dist_in1kztimm/)	hf_hub_idzlevit_128.fb_dist_in1kzlevit_192.fb_dist_in1kzlevit_256.fb_dist_in1kzlevit_384.fb_dist_in1kzlevit_conv_128s.fb_dist_in1k)r   r   )r  r  zlevit_conv_128.fb_dist_in1kzlevit_conv_192.fb_dist_in1kzlevit_conv_256.fb_dist_in1kzlevit_conv_384.fb_dist_in1kzlevit_384_s8.untrainedr  )r  zlevit_512_s8.untrainedzlevit_512.untrainedzlevit_256d.untrainedzlevit_512d.untrainedzlevit_conv_384_s8.untrainedzlevit_conv_512_s8.untrained)zlevit_conv_512.untrainedzlevit_conv_256d.untrainedzlevit_conv_512d.untrainedr   c                     [        SSU 0UD6$ )Nr
  )r  r  r
  r  s     r9   r  r    s    FFvFFr;   c                     [        SSU 0UD6$ )Nr
  )r  r  r  s     r9   r  r  #      E
EfEEr;   c                     [        SSU 0UD6$ )Nr
  )r  r  r  s     r9   r  r  (  r!  r;   c                     [        SSU 0UD6$ )Nr
  )r  r  r  s     r9   r  r  -  r!  r;   c                     [        SSU 0UD6$ )Nr
  )r  r  r  s     r9   r  r  2  r!  r;   c                     [        SSU 0UD6$ )Nr
  )r  r  r  s     r9   r  r  7  s    H:HHHr;   c                     [        SU SS.UD6$ )NFr
  r  )r  r  r  s     r9   r  r  <  s    Y:YRXYYr;   c                     [        SU SS.UD6$ )NFr'  )r  r  r  s     r9   r  r  A  s    V
eVvVVr;   c                     [        SU SS.UD6$ )NFr'  )r  r  r  s     r9   r  r  F      WuWPVWWr;   c                     [        SU SS.UD6$ )NFr'  )r  r  r  s     r9   r  r  K  r*  r;   c                     [        SU SS.UD6$ )NTr
  r   )levit_conv_128sr  r  s     r9   r.  r.  P  s    Zj4ZSYZZr;   c                     [        SU SS.UD6$ )NTr-  )levit_conv_128r  r  s     r9   r0  r0  U      YZ$YRXYYr;   c                     [        SU SS.UD6$ )NTr-  )levit_conv_192r  r  s     r9   r3  r3  Z  r1  r;   c                     [        SU SS.UD6$ )NTr-  )levit_conv_256r  r  s     r9   r5  r5  _  r1  r;   c                     [        SU SS.UD6$ )NTr-  )levit_conv_384r  r  s     r9   r7  r7  d  r1  r;   c                     [        SU SS.UD6$ )NTr-  )levit_conv_384_s8r  r  s     r9   r9  r9  i  s    \
T\U[\\r;   c                      [        SU SSS.UD6$ )NTFr
  r   r  )levit_conv_512_s8r  r  s     r9   r<  r<  n  s    m
T]bmflmmr;   c                      [        SU SSS.UD6$ )NTFr;  )levit_conv_512r  r  s     r9   r>  r>  s  s    jZ$Z_jcijjr;   c                      [        SU SSS.UD6$ )NTFr;  )levit_conv_256dr  r  s     r9   r@  r@  x      kj4[`kdjkkr;   c                      [        SU SSS.UD6$ )NTFr;  )levit_conv_512dr  r  s     r9   rC  rC  }  rA  r;   )NFT)rT  r  )Nr:  collectionsr   	functoolsr   typingr   r   r   r   r	   r
   rZ   torch.nnr.   	timm.datar   r   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__r   r   r`   rp   r   r   r   r   r   r  r2  r<  rK  rS  r   r  r  r{  r  r  r  default_cfgsr  r  r  r  r  r  r  r  r  r  r.  r0  r3  r5  r7  r9  r<  r>  r@  rC  rB  r;   r9   <module>rP     s  2 $  ; ;   A \ \ * + 3 <)#'ryy #'L6 6@%2 %2P`BMM `(`R]] `,# #0l		 l^@")) @Fryy B/bii /d, ,^? ?DWBII Wt"$U "$J* !2)U!29V!2)U!2)U!29V !29D* !2ID*
 "B+Ybhj !2)_eg!2Iagi7
@.	 % 3&t 3&
 d3& d3& d3& d3&& #D%'3&. "4$/3&6 "4$73&> "4$?3&F "4$G3&P dm<Q3&R dm<S3&T 4=9U3&V DM:W3&X DM:Y3&\ "4=#A]3&^ "4=#A_3&` !% >!%!?!%!?e3& 3l Ge G G FU F F FU F F FU F F FU F F I I I Z Z Z WU W W Xe X X Xe X X [5 [ [ Z% Z Z Z% Z Z Z% Z Z Z% Z Z ]U ] ] nU n n k% k k l5 l l l5 l lr;   