
    RЦi]              	          S r SSKrSSKJrJrJrJrJrJrJ	r	J
r
  SSKrSSKJr  SSKJs  Jr  SSKJrJr  SSKJrJrJrJrJr  SSKJr  SSKJr  SS	KJr  SS
K J!r!J"r"  S/r# " S S\RH                  5      r% " S S\RH                  5      r& " S S\RH                  5      r' " S S\RH                  5      r( " S S\RH                  5      r) " S S\RH                  5      r* " S S\RH                  5      r+ " S S\RH                  5      r, " S S\RH                  5      r-S\\.\R^                  4   S\RH                  S\\.\R^                  4   4S  jr0S.S!\.S"\S\\.\4   4S# jjr1\!" \1" S$S%9\1" S$S%9\1" S$S%9\1" S$S%9S&.5      r2S/S'\.S(\3S"\S\-4S) jjr4\"S/S(\3S"\S\-4S* jj5       r5\"S/S(\3S"\S\-4S+ jj5       r6\"S/S(\3S"\S\-4S, jj5       r7\"S/S(\3S"\S\-4S- jj5       r8g)0aj  SwiftFormer
SwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applications
Code: https://github.com/Amshaker/SwiftFormer
Paper: https://arxiv.org/pdf/2303.15446

@InProceedings{Shaker_2023_ICCV,
    author    = {Shaker, Abdelrahman and Maaz, Muhammad and Rasheed, Hanoona and Khan, Salman and Yang, Ming-Hsuan and Khan, Fahad Shahbaz},
    title     = {SwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applications},
    booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
    year      = {2023},
}
    N)AnyDictListOptionalSetTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathLinear	LayerType	to_2tupletrunc_normal_   )build_model_with_cfg)feature_take_indices)checkpoint_seq)generate_default_cfgsregister_modelSwiftFormerc                   v   ^  \ rS rSrS
S\S\S\4U 4S jjjrS\R                  S\R                  4S jr
S	rU =r$ )LayerScale2d   diminit_valuesinplacec                    > XES.n[         TU ]  5         X0l        [        R                  " U[
        R                  " USS40 UD6-  SS9U l        g )Ndevicedtyper   T)requires_grad)super__init__r   nn	Parametertorchonesgamma)selfr   r   r   r"   r#   dd	__class__s          V/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/swiftformer.pyr&   LayerScale2d.__init__   sH    /\\%**S!Q5"55TK
    xreturnc                 v    U R                   (       a  UR                  U R                  5      $ XR                  -  $ N)r   mul_r+   r,   r2   s     r/   forwardLayerScale2d.forward&   s&    %)\\qvvdjj!Eq::~Er1   )r+   r   )h㈵>FNN)__name__
__module____qualname____firstlineno__intfloatboolr&   r)   Tensorr8   __static_attributes____classcell__r.   s   @r/   r   r      sK    KC Ke KT K KF F%,, F Fr1   r   c                      ^  \ rS rSrSrSSSSS\R                  SS4S\S	\S
\S\S\S\\R                     4U 4S jjjr
S\R                  S\R                  4S jrSrU =r$ )	Embedding*   z
Patch Embedding that is implemented by a layer of conv.
Input: tensor in shape [B, C, H, W]
Output: tensor in shape [B, C, H/stride, W/stride]
   i      r   Nin_chans	embed_dim
patch_sizestridepadding
norm_layerc	                   > XxS.n	[         T
U ]  5         [        U5      n[        U5      n[        U5      n[        R                  " XX4U40 U	D6U l        U(       a  U" U40 U	D6U l        g [        R                  " 5       U l        g )Nr!   )r%   r&   r   r'   Conv2dprojIdentitynorm)r,   rK   rL   rM   rN   rO   rP   r"   r#   r-   r.   s             r/   r&   Embedding.__init__0   so     /z*
6"G$IIh:wURTU	3=Jy/B/	2;;=	r1   r2   r3   c                 J    U R                  U5      nU R                  U5      nU$ r5   )rS   rU   r7   s     r/   r8   Embedding.forwardC   s!    IIaLIIaLr1   )rU   rS   )r;   r<   r=   r>   __doc__r'   BatchNorm2dr?   r	   Moduler&   r)   rB   r8   rC   rD   rE   s   @r/   rG   rG   *   s       *,..QQ Q 	Q
 Q Q RYYQ Q& %,,  r1   rG   c                      ^  \ rS rSrSrSSS\R                  \R                  SSS4S\S	\S
\S\	S\
\R                     S\
\R                     S\4U 4S jjjrS\R                  S\R                  4S jrSrU =r$ )ConvEncoderI   z
Implementation of ConvEncoder with 3*3 and 1*1 convolutions.
Input: tensor with shape [B, C, H, W]
Output: tensor with shape [B, C, H, W]
@   rI           TNr   
hidden_dimkernel_size	drop_path	act_layerrP   use_layer_scalec
                   > XS.n
[         TU ]  5         [        R                  " XU4US-  US.U
D6U l        U" U40 U
D6U l        [        R                  " XS40 U
D6U l        U" 5       U l        [        R                  " X!S40 U
D6U l        US:  a  [        U5      O[        R                  " 5       U l        U(       a  [        US40 U
D6U l        g [        R                  " 5       U l        g )Nr!      rO   groupsr   r`   r%   r&   r'   rR   dwconvrU   pwconv1actpwconv2r   rT   rc   r   layer_scale)r,   r   ra   rb   rc   rd   rP   re   r"   r#   r-   r.   s              r/   r&   ConvEncoder.__init__O   s     /ii+b{a?OX[b_abs)b)	yy!:r:;yy!:r:09B),BKKM9H<Q5"5bkkmr1   r2   r3   c                     UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nX R                  U5      -   nU$ r5   rk   rU   rl   rm   rn   ro   rc   )r,   r2   inputs      r/   r8   ConvEncoder.forwarde   sn    KKNIIaLLLOHHQKLLOQNN1%%r1   rm   rc   rk   ro   rU   rl   rn   r;   r<   r=   r>   rY   r'   GELUrZ   r?   r@   r	   r[   rA   r&   r)   rB   r8   rC   rD   rE   s   @r/   r]   r]   I   s     ! !)+*,..$(\\ \ 	\
 \ BII\ RYY\ "\ \,	 	%,, 	 	r1   r]   c                      ^  \ rS rSrSrSS\R                  \R                  SSS4S\S\	\   S\	\   S\
\R                     S	\
\R                     S
\4U 4S jjjrS\R                  S\R                  4S jrSrU =r$ )Mlpq   z
Implementation of MLP layer with 1*1 convolutions.
Input: tensor with shape [B, C, H, W]
Output: tensor with shape [B, C, H, W]
Nr`   in_featureshidden_featuresout_featuresrd   rP   dropc	                 4  > XxS.n	[         T
U ]  5         U=(       d    UnU=(       d    UnU" U40 U	D6U l        [        R                  " XS40 U	D6U l        U" 5       U l        [        R                  " X#S40 U	D6U l        [        R                  " U5      U l	        g )Nr!   r   )
r%   r&   norm1r'   rR   fc1rm   fc2Dropoutr~   )r,   r{   r|   r}   rd   rP   r~   r"   r#   r-   r.   s             r/   r&   Mlp.__init__w   s     /#2{)8[2r2
99[1CC;99_ADDJJt$	r1   r2   r3   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU$ r5   )r   r   rm   r~   r   r7   s     r/   r8   Mlp.forward   sU    JJqMHHQKHHQKIIaLHHQKIIaLr1   )rm   r~   r   r   r   )r;   r<   r=   r>   rY   r'   rw   rZ   r?   r   r	   r[   r@   r&   r)   rB   r8   rC   rD   rE   s   @r/   ry   ry   q   s     .2*.)+*,..%% &c]% #3-	%
 BII% RYY% % %* %,,  r1   ry   c                      ^  \ rS rSrSr     SS\S\S\4U 4S jjjrS\R                  S\R                  4S	 jr	S
r
U =r$ )EfficientAdditiveAttention   z
Efficient Additive Attention module for SwiftFormer.
Input: tensor in shape [B, C, H, W]
Output: tensor in shape [B, C, H, W]
in_dims	token_dim	num_headsc                   > XES.n[         TU ]  5         US-  U l        [        R                  " XU-  40 UD6U l        [        R                  " XU-  40 UD6U l        [        R                  " [        R                  " X#-  S40 UD65      U l
        [        R                  " X#-  X#-  40 UD6U l        [        R                  " X#-  U40 UD6U l        g )Nr!   g      r   )r%   r&   scale_factorr'   r   to_queryto_keyr(   r)   randnw_grS   final)r,   r   r   r   r"   r#   r-   r.   s          r/   r&   #EfficientAdditiveAttention.__init__   s     /%-		'y+@GBGiiY)>E"E<<I,A1 K KLIIi3Y5JQbQ	YYy4iF2F
r1   r2   r3   c                    UR                   u  p#pEUR                  S5      R                  SSS5      n[        R                  " U R                  U5      SS9n[        R                  " U R                  U5      SS9n[        R                  " X`R                  -  U R                  -  SS9n[        R                  " X-  SSS9nU R                  X-  5      U-   n	U R                  U	5      R                  SSS5      R                  USXE5      n	U	$ )Nrg   r   r   r   T)r   keepdim)shapeflattenpermuteF	normalizer   r   r   r   r)   sumrS   r   reshape)
r,   r2   B_HWquerykeyattnouts
             r/   r8   "EfficientAdditiveAttention.forward   s    WW
aIIaL  Aq)DMM!,"5kk$++a.b1{{588+d.?.??QGyy1d;ii
#e+jjo%%aA.66q"aC
r1   )r   rS   r   r   r   r   )      r   NN)r;   r<   r=   r>   rY   r?   r&   r)   rB   r8   rC   rD   rE   s   @r/   r   r      sd      GG G 	G G& %,,  r1   r   c                      ^  \ rS rSrSrSSS\R                  \R                  SS4S\S\S	\	S
\
S\\R                     S\\R                     4U 4S jjjrS\R                  S\R                  4S jrSrU =r$ )LocalRepresentation   z
Local Representation module for SwiftFormer that is implemented by 3*3 depth-wise and point-wise convolutions.
Input: tensor in shape [B, C, H, W]
Output: tensor in shape [B, C, H, W]
rI   r`   TNr   rb   rc   re   rd   rP   c	                   > XxS.n	[         T
U ]  5         [        R                  " XU4US-  US.U	D6U l        U" U40 U	D6U l        [        R                  " X4SS0U	D6U l        U" 5       U l        [        R                  " X4SS0U	D6U l        US:  a  [        U5      O[        R                  " 5       U l        U(       a  [        US40 U	D6U l        g [        R                  " 5       U l        g )Nr!   rg   rh   rb   r   r`   rj   )r,   r   rb   rc   re   rd   rP   r"   r#   r-   r.   s             r/   r&   LocalRepresentation.__init__   s     /ii+b{a?OX[b_abs)b)	yy?q?B?;yy?q?B?09B),BKKM9H<Q5"5bkkmr1   r2   r3   c                     UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nX R                  U5      -   nU$ r5   rr   )r,   r2   skips      r/   r8   LocalRepresentation.forward   sn    KKNIIaLLLOHHQKLLOQ>>!$$r1   ru   )r;   r<   r=   r>   rY   r'   rw   rZ   r?   r@   rA   r	   r[   r&   r)   rB   r8   rC   rD   rE   s   @r/   r   r      s      !!$()+*,..\\ \ 	\
 "\ BII\ RYY\ \*	 	%,, 	 	r1   r   c                      ^  \ rS rSrSrSSS\R                  \R                  SSSS4	S\S	\	S
\	S\	S\
\R                     S\
\R                     S\S\	4U 4S jjjrS\R                  S\R                  4S jrSrU =r$ )Block   z
SwiftFormer Encoder Block for SwiftFormer. It consists of :
(1) Local representation module, (2) EfficientAdditiveAttention, and (3) MLP block.
Input: tensor in shape [B, C, H, W]
Output: tensor in shape [B, C, H, W]
      @r`   Tr:   Nr   	mlp_ratio	drop_raterc   rd   rP   re   layer_scale_init_valuec           	        > XS.n[         TU ]  5         [        SUUUUS.UD6U l        [	        SXS.UD6U l        [        SU[        X-  5      UUUS.UD6U l        US:  a  [        U5      O[        R                  " 5       U l        U(       a  [        X40 UD6O[        R                  " 5       U l        U(       a  [        X40 UD6U l        g [        R                  " 5       U l        g )Nr!   )r   re   rd   rP   )r   r   )r{   r|   rd   rP   r~   r`    )r%   r&   r   local_representationr   r   ry   r?   linearr   r'   rT   rc   r   layer_scale_1layer_scale_2)r,   r   r   r   rc   rd   rP   re   r   r"   r#   r-   r.   s               r/   r&   Block.__init__   s     /$7 %
+!	%

 %
! /PsPRP	 
0!
 
 1:B),BKKM *#LL$&KKM 	  *#LL$&KKM 	r1   r2   r3   c                     U R                  U5      nXR                  U R                  U R                  U5      5      5      -   nXR                  U R	                  U R                  U5      5      5      -   nU$ r5   )r   rc   r   r   r   r   r7   s     r/   r8   Block.forward  s_    %%a(t11$))A,?@@t11$++a.ABBr1   )r   rc   r   r   r   r   rv   rE   s   @r/   r   r      s      "!!)+*,..$(,0#2#2 #2 	#2
 #2 BII#2 RYY#2 "#2 %*#2 #2J %,,  r1   r   c                   0  ^  \ rS rSrSrS\R                  \R                  SSSSSSS4
S\S	\S
\	\   S\
S\\R                     S\\R                     S\
S\
S\S\
S\\\R                        4U 4S jjjrS\R"                  S\R"                  4S jrSrU =r$ )Stagei  z
Implementation of each SwiftFormer stages. Here, SwiftFormerEncoder used as the last block in all stages, while ConvEncoder used in the rest of the blocks.
Input: tensor in shape [B, C, H, W]
Output: tensor in shape [B, C, H, W]
r   r`   Tr:   Nr   indexlayersr   rd   rP   r   drop_path_ratere   r   
downsamplec                   > XS.n[         TU ]  5         SU l        Ub  UO[        R                  " 5       U l        / n[        X2   5       H  nUU[        US U 5      -   -  [        U5      S-
  -  nX2   U-
  S::  a%  UR                  [        U4UUUUUU	U
S.UD65        MV  UR                  [        SU[        XA-  5      SUUUU	S.UD65        M     [        R                  " U6 U l        g )Nr!   Fr   )r   r   rc   rd   rP   re   r   rI   )r   ra   rb   rc   rd   rP   re   r   )r%   r&   grad_checkpointingr'   rT   r   ranger   appendr   r]   r?   
Sequentialblocks)r,   r   r   r   r   rd   rP   r   r   re   r   r   r"   r#   r-   r   	block_idx	block_dprr.   s                     r/   r&   Stage.__init__  s     /"'(2(>*BKKMv}-I&)c&%.6I*IJcRXk\]o^I}y(A-e
'''')$3+A
 
 
 k 	"9?3 !'')$3	 	 	 .2 mmV,r1   r2   r3   c                     U R                  U5      nU R                  (       a;  [        R                  R	                  5       (       d  [        U R                  U5      nU$ U R                  U5      nU$ r5   )r   r   r)   jitis_scriptingr   r   r7   s     r/   r8   Stage.forwardO  sV    OOA""599+A+A+C+Ct{{A.A  AAr1   )r   r   r   )r;   r<   r=   r>   rY   r'   rw   rZ   r?   r   r@   r	   r[   rA   r   r&   r)   rB   r8   rC   rD   rE   s   @r/   r   r     s      ")+*,..!$&$(,048/-/- /- I	/-
 /- BII/- RYY/- /- "/- "/- %*/- !bii1/- /-b %,,  r1   r   c            !         ^  \ rS rSr/ SQ/ SQS/ SQ\R
                  SSSS	S
S
SSSSSSS4S\\   S\\   S\S\\   S\	\R                     S\S\S\S\S\S\S\S\S\S\S\4 U 4S  jjjrS! r\R                   R"                  S"\4S# j5       r\R                   R"                  S:S$\S"\\\4   4S% jj5       r\R                   R"                  S;S&\4S' jj5       r\R                   R"                  S"\\R                  \R                  4   4S( j5       rS<S\S\\   4S) jjr\R                   R"                  S;S&\4S* jj5       r     S=S+\R:                  S,\\\\\   4      S-\S.\S/\S0\S"\\\R:                     \\R:                  \\R:                     4   4   4S1 jjr   S>S,\\\\   4   S2\S3\4S4 jjr S+\R:                  S"\R:                  4S5 jr!S:S+\R:                  S6\4S7 jjr"S+\R:                  4S8 jr#S9r$U =r%$ )?r   iX  rI   rI         0   8   p      r   )FTTTrI   rg   r     r`   Tr:   avg    Nr   
embed_dims
mlp_ratiosdownsamplesrd   down_patch_sizedown_stridedown_padnum_classesr   r   re   r   global_pooloutput_striderK   c                   > [         TU ]  5         UUS.nUS:X  d   eXl        UU l        Xl        / U l        [        R                  " [        R                  " UUS   S-  SSS40 UD6[        R                  " US   S-  40 UD6[        R                  " 5       [        R                  " US   S-  US   SSS40 UD6[        R                  " US   40 UD6[        R                  " 5       5      U l        US   n/ n[        [        U5      5       H  nUU   (       a  [        SUUU   UUUS.UD6O[        R                  " 5       n[!        SUU   UUUUU
UUUUS.
UD6nUU   nUR#                  U5        U =R
                  [%        UU   SUS-   -  S	U 3S
9/-  sl        M     [        R                  " U6 U l        US   =U l        =U l        n[        R                  " U40 UD6U l        [        R.                  " U
5      U l        U	S:  a  [3        UU	40 UD6O[        R                  " 5       U l        U	S:  a  [3        UU	40 UD6O[        R                  " 5       U l        SU l        U R;                  5         g )Nr!   r   r   rg   rI   r   )rK   rL   rM   rN   rO   )
r   r   r   r   rd   r   r   re   r   r   stages.)num_chs	reductionmoduler   Fr   )r%   r&   r   rK   r   feature_infor'   r   rR   rZ   ReLUstemr   lenrG   rT   r   r   dictstagesnum_featureshead_hidden_sizerU   r   	head_dropr   head	head_distdistilled_training_initialize_weights)r,   r   r   r   r   rd   r   r   r   r   r   r   re   r   r   r   rK   r"   r#   kwargsr-   prev_dimr   ir   stageout_chsr.   s                              r/   r&   SwiftFormer.__init__Y  sy   , 	/"""& &MMIIh
1 2Aq!BrBNN:a=A-44GGIIIjmq(*Q-AqGBGNN:a=/B/GGI
	 a=s6{#A Q # !$Q-*"   &([[]   qM$##- /'=% E "!}HMM% $z!}AaC[bcdbeYf"g!hh1 $2 mmV, @J"~MMT2WNN71b1	I.:E/F7K626r{{}	?JQ;;TVT_T_Ta"'  "r1   c                    U R                  5        H  u  p[        U[        R                  5      (       aO  [	        UR
                  SS9  UR                  b,  [        R                  R                  UR                  S5        Mq  Ms  [        U[        R                  5      (       d  M  [	        UR
                  SS9  UR                  c  M  [        R                  R                  UR                  S5        M     g )Ng{Gz?)stdr   )
named_modules
isinstancer'   r   r   weightbiasinit	constant_rR   )r,   namems      r/   r   SwiftFormer._initialize_weights  s    ))+GD!RYY''ahhC066%GG%%affa0 &Aryy))ahhC066%GG%%affa0 ,r1   r3   c                     [        5       $ r5   )setr,   s    r/   no_weight_decaySwiftFormer.no_weight_decay  s	    ur1   coarsec                 0    [        SU(       a  SO/ SQS9nU$ )Nz^stemz^stages\.(\d+)))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)z^norm)i )r   r   )r   )r,   r  matchers      r/   group_matcherSwiftFormer.group_matcher  s!    (.$ 5
 r1   enablec                 6    U R                    H	  nXl        M     g r5   )r   r   )r,   r  ss      r/   set_grad_checkpointing"SwiftFormer.set_grad_checkpointing  s    A#)  r1   c                 2    U R                   U R                  4$ r5   r   r   r  s    r/   get_classifierSwiftFormer.get_classifier  s    yy$..((r1   c                    Xl         Ub  X l        U R                  R                  R                  [        U R                  S5      (       a   U R                  R                  R                  OSpCUS:  a  [        U R                  XUS9O[        R                  " 5       U l        US:  a  [        U R                  XUS9U l        g [        R                  " 5       U l        g )Nr  )NNr   r!   )r   r   r   r  r"   hasattrr#   r   r   r'   rT   r   )r,   r   r   r"   r#   s        r/   reset_classifierSwiftFormer.reset_classifier  s    &"*		((//7SWS\S\^fKgKg1A1A1G1GmyZehiZiF4,,kPUVoqozozo|	_jmn_n 1 1;UZ[tvtt  uBr1   c                     Xl         g r5   )r   )r,   r  s     r/   set_distilled_training"SwiftFormer.set_distilled_training  s    "(r1   r2   indicesrU   
stop_early
output_fmtintermediates_onlyc                 4   US;   d   S5       e/ n[        [        U R                  5      U5      u  p[        U R                  5      S-
  n
U R                  U5      n[        R
                  R                  5       (       d  U(       d  U R                  nOU R                  SU	S-    n[        U5       HE  u  pU" U5      nX;   d  M  U(       a  X:X  a  U R                  U5      nOUnUR                  U5        MG     U(       a  U$ WU
:X  a  U R                  U5      nX4$ )a  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    norm: Apply norm layer to compatible intermediates
    stop_early: Stop iterating over blocks when last desired intermediate hit
    output_fmt: Shape of intermediate feature outputs
    intermediates_only: Only return intermediate features
Returns:

)NCHWzOutput shape must be NCHW.r   N)
r   r   r   r   r)   r   r   	enumeraterU   r   )r,   r2   r(  rU   r)  r*  r+  intermediatestake_indices	max_indexlast_idxr   feat_idxr   x_inters                  r/   forward_intermediates!SwiftFormer.forward_intermediates  s    * Y&D(DD&"6s4;;7G"Qt{{#a' IIaL99!!##:[[F[[)a-0F(0OHaA'H0"iilGG$$W-  1   x		!Ar1   
prune_norm
prune_headc                     [        [        U R                  5      U5      u  pEU R                  SUS-    U l        U(       a  [        R                  " 5       U l        U(       a  U R                  SS5        U$ )z?Prune layers not required for specified intermediates.
        Nr   r    )r   r   r   r'   rT   rU   r#  )r,   r(  r7  r8  r0  r1  s         r/   prune_intermediate_layers%SwiftFormer.prune_intermediate_layers	  s[     #7s4;;7G"Qkk.9q=1DI!!!R(r1   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ r5   )r   r   rU   r7   s     r/   forward_featuresSwiftFormer.forward_features  s.    IIaLKKNIIaLr1   
pre_logitsc                 T   U R                   S:X  a  UR                  SS9nU R                  U5      nU(       a  U$ U R                  U5      U R	                  U5      p1U R
                  (       a7  U R                  (       a&  [        R                  R                  5       (       d  X4$ X-   S-  $ )Nr   )rg   rI   r   rg   )
r   meanr   r   r   r   trainingr)   r   r   )r,   r2   r@  x_dists       r/   forward_headSwiftFormer.forward_head  s    u$6"ANN1HIIaL$.."36""t}}UYY=S=S=U=U9 J!##r1   c                 J    U R                  U5      nU R                  U5      nU$ r5   )r>  rE  r7   s     r/   r8   SwiftFormer.forward-  s'    !!!$a r1   )r   r   r   r   r   r   r   rK   rU   r   r   r   r   F)Tr5   )NFFr-  F)r   FT)&r;   r<   r=   r>   r'   rw   r   r?   rA   r	   r[   r@   strr&   r   r)   r   ignorer   r  r   r   r  r  r   r  r   r#  r&  rB   r
   r5  r;  r>  rE  r8   rC   rD   rE   s   @r/   r   r   X  s"    !-$6&?)+#$ #!$&$(,0$!#'L#IL# S	L# 	L#
 dL# BIIL# !L# L# L# L# L# "L# "L# %*L# L#  !L#" #L# L#\	1 YY   YY	D 	T#s(^ 	 	 YY*T * * YY)bii&: ; ) )BC Bhsm B YY)T ) ) 8<$$',0 ||0  eCcN340  	0 
 0  0  !%0  
tELL!5tELL7I)I#JJ	K0 h ./$#	3S	>*  	 %,, 5<< $ell $ $  r1   
state_dictmodelr3   c                 0   U R                  SU 5      n SU ;   a  U $ 0 nU R                  5        H  u  p4UR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  S	S
5      nUR                  SS5      n[        R                  " SSU5      n[        R
                  " SU5      nU(       aL  [        UR                  S5      5      UR                  S5      pvUS-  nUS-  S:X  a	  SU SU 3nOSUS-    SU 3nXBU'   M     U$ )NrM  zstem.0.weightzpatch_embed.zstem.z
dist_head.z
head_dist.z
attn.Proj.z
attn.proj.z.layer_scale_1z.layer_scale_1.gammaz.layer_scale_2z.layer_scale_2.gammaz\.layer_scale(?=$|\.)z.layer_scale.gammaz^network\.(\d+)\.(.*)r   rg   r   r   z.blocks.z.downsample.)getitemsreplaceresubmatchr?   group)	rL  rM  out_dictkvr  n_idxrest	stage_idxs	            r/   checkpoint_filter_fnr\  3  s   4J*$H  "IIng.IIlL1IIlL1II&(>?II&(>?FF+-A1EHH-q1aggaj/1771:4
IqyA~i[7ik],tf=! #" Or1   urlr   c                 :    U SSS SSS[         [        SSSS	S
SS.UE$ )Nr   )rI      r_  Tgffffff?bicubiczstem.0r  z
apache-2.0zarXiv:2303.15446zdSwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applicationsz'https://github.com/Amshaker/SwiftFormer)r]  r   
input_size	pool_sizefixed_input_sizecrop_pctinterpolationrB  r  
first_conv
classifierlicense	paper_ids
paper_name
origin_urlr   )r]  r   s     r/   _cfgrl  M  s@    =tae)%.B.C'|?  r1   ztimm/)	hf_hub_id)zswiftformer_xs.dist_in1kzswiftformer_s.dist_in1kzswiftformer_l1.dist_in1kzswiftformer_l3.dist_in1kvariant
pretrainedc           	      F    [        [        X4[        [        SSS9S.UD6nU$ )N)r   r   rg   rI   T)out_indicesflatten_sequential)pretrained_filter_fnfeature_cfg)r   r   r\  r   )rn  ro  r   rM  s       r/   _create_swiftformerru  l  s4     W1\dK 	E Lr1   c           	      L    [        / SQ/ SQS9n[        SSU 0[        U40 UD6D6$ )Nr   r   r   r   ro  )swiftformer_xsr   ru  ro  r   
model_argss      r/   rx  rx  v  s-    \6HIJeJe$zJd]cJdeer1   c           	      L    [        / SQ/ SQS9n[        SSU 0[        U40 UD6D6$ )N)rI   rI   	   r   )r   r_      r_  rw  ro  )swiftformer_sry  rz  s      r/   r  r  |  s-    \6HIJd:djIc\bIcddr1   c           	      L    [        / SQ/ SQS9n[        SSU 0[        U40 UD6D6$ )N)r   rI   
      )r   `      i  rw  ro  )swiftformer_l1ry  rz  s      r/   r  r    s-    ]7IJJeJe$zJd]cJdeer1   c           	      L    [        / SQ/ SQS9n[        SSU 0[        U40 UD6D6$ )N)r   r      r   )r_      i@  r   rw  ro  )swiftformer_l3ry  rz  s      r/   r  r    s-    ]7JKJeJe$zJd]cJdeer1   )r:  rI  )9rY   rR  typingr   r   r   r   r   r   r	   r
   r)   torch.nnr'   torch.nn.functional
functionalr   	timm.datar   r   timm.layersr   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r[   r   rG   r]   ry   r   r   r   r   r   rJ  rB   r\  rl  default_cfgsrA   ru  rx  r  r  r  r   r1   r/   <module>r     sY   
 E E E     A M M * + ' </	F299 	F		 >%")) %P"")) "J% %P$")) $N0BII 0f=BII =@X")) XvT#u||*;%< RYY SWX[]b]i]iXiSj 4c # $sCx.  % $!  $  !%! !%!&   $ # R]  ft fs f{ f f
 ed ec ek e e ft fs f{ f f
 ft fs f{ f fr1   