
    RЦi{                        S r SSKJrJrJrJrJrJr  SSKrSSK	J
r
  SSKJ
s  Jr  SSKJrJr  SSKJrJrJrJrJrJrJr  SSKJr  SSKJrJr  S	/r " S
 S\
R>                  5      r  " S S\
R>                  5      r! " S S\
R>                  5      r" " S S\
R>                  5      r# " S S\
R>                  5      r$ " S S	\
R>                  5      r%S r&S r'S r(S*S jr)S+S jr*\" \*" SS9\*" SS9\*" SS9\*" SS9\*" SS9\*" SS9\*" SS9\*" SSSSS9S .5      r+\S,S!\%4S" jj5       r,\S,S!\%4S# jj5       r-\S,S!\%4S$ jj5       r.\S,S!\%4S% jj5       r/\S,S!\%4S& jj5       r0\S,S!\%4S' jj5       r1\S,S!\%4S( jj5       r2\S,S!\%4S) jj5       r3g)-z
CoaT architecture.

Paper: Co-Scale Conv-Attentional Image Transformers - https://arxiv.org/abs/2104.06399

Official CoaT code at: https://github.com/mlpc-ucsd/CoaT

Modified from timm/models/vision_transformer.py
    )ListOptionalTupleUnionTypeAnyNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPath	to_2tupletrunc_normal__assert	LayerNorm   )build_model_with_cfg)register_modelgenerate_default_cfgsCoaTc            	       f   ^  \ rS rSrSr  S
S\S\S\\\4   4U 4S jjjrS\	\\4   4S jr
S	rU =r$ )ConvRelPosEnc   z*Convolutional relative position encoding. head_chs	num_headswindowc           	      V  > XES.n[         TU ]  5         [        U[        5      (       a
  X20nX0l        O&[        U[
        5      (       a  X0l        O
[        5       e[        R                  " 5       U l	        / U l
        UR                  5        Hs  u  pxSn	XwS-
  U	S-
  -  -   S-  n
[        R                  " X-  X-  4Xw4X4X4X-  S.UD6nU R                  R                  U5        U R                  R                  U5        Mu     U R                   Vs/ s H  oU-  PM	     snU l        gs  snf )a  
Initialization.
    Ch: Channels per head.
    h: Number of heads.
    window: Window size(s) in convolutional relative positional encoding. It can have two forms:
        1. An integer of window size, which assigns all attention heads with the same window s
            size in ConvRelPosEnc.
        2. A dict mapping window size to #attention head splits (
            e.g. {window size 1: #attention head split 1, window size 2: #attention head split 2})
            It will apply different window size to the attention head splits.
devicedtyper      )kernel_sizepaddingdilationgroupsN)super__init__
isinstanceintr   dict
ValueErrornn
ModuleList	conv_listhead_splitsitemsConv2dappendchannel_splits)selfr   r   r   r    r!   dd
cur_windowcur_head_splitr%   padding_sizecur_convx	__class__s                O/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/coat.pyr(   ConvRelPosEnc.__init__   s&   & /fc""(F K%% K,*0,,.&JH 'q.X\)JJqPLyy)) (4%4"-%0 H NN!!(+##N3 +9  6:5E5EF5E8|5EFFs   D&sizec                 >   UR                   u  pEpgUu  p[        USX-  -   :H  S5        US S 2S S 2SS 2S S 24   n
US S 2S S 2SS 2S S 24   nUR                  SS5      R                  XEU-  X5      n[        R
                  " XR                  SS9n/ n[        U R                  5       H  u  pUR                  U" X   5      5        M      [        R                  " USS9nUR                  XEXxU	-  5      R                  SS5      nU
U-  n[        R                  " US5      nU$ )Nr    dim)r   r   r   r   r   r   )shaper   	transposereshapetorchsplitr4   	enumerater/   r3   catFpad)r5   qvr?   Br   NCHWq_imgv_img
v_img_listconv_v_img_listiconv
conv_v_imgEV_hats                     r=   forwardConvRelPosEnc.forwardM   s   WWaQY# !QA+!QA+B'//q=!G[[(;(;C
 0GA""4
#67 1YYA6
''aQ?II"bQ
#v12    )r4   r/   r0   r   )NN)__name__
__module____qualname____firstlineno____doc__r*   r   r+   r(   r   r^   __static_attributes____classcell__r<   s   @r=   r   r      s\    5 1G1G 1G #t)$	1G 1Gf%S/  r`   r   c                   x   ^  \ rS rSrSr       SS\S\S\S\S\S\\	   4U 4S	 jjjr
S
\\\4   4S jrSrU =r$ )FactorAttnConvRelPosEncc   zJFactorized attention with convolutional relative position encoding class. rE   r   qkv_bias	attn_drop	proj_dropshared_crpec	                 D  > XxS.n	[         TU ]  5         X l        X-  n
U
S-  U l        [        R
                  " XS-  4SU0U	D6U l        [        R                  " U5      U l        [        R
                  " X40 U	D6U l	        [        R                  " U5      U l
        X`l        g )Nr   g         bias)r'   r(   r   scaler-   LinearqkvDropoutrm   projrn   crpe)r5   rE   r   rl   rm   rn   ro   r    r!   r6   head_dimr<   s              r=   r(    FactorAttnConvRelPosEnc.__init__e   s     /"#%
99S'??B?I.IIc-"-	I.  	r`   r?   c                     UR                   u  p4nU R                  U5      R                  X4SU R                  XPR                  -  5      R	                  SSSSS5      nUR                  S5      u  pxn	UR                  SS9n
U
R                  SS5      U	-  nX{-  nU R                  XyUS	9nU R                  U-  U-   nUR                  SS5      R                  X4U5      nU R                  U5      nU R                  U5      nU$ )
Nrq   r"   r   r      rD   rB   rC   r?   )rF   ru   rH   r   permuteunbindsoftmaxrG   rx   rs   rw   rn   )r5   r;   r?   rQ   rR   rS   ru   rO   krP   	k_softmax
factor_attrx   s                r=   r^   FactorAttnConvRelPosEnc.forward~   s    ''a hhqk!!!4>>1;NOWWXY[\^_abdef**Q-a II!I$	((R014
^
 yyDy) JJ#d*KK1%%aA. IIaLNN1r`   )rm   rx   r   rw   rn   ru   rs   )   F        r   NNN)ra   rb   rc   rd   re   r*   boolfloatr   r   r(   r   r^   rf   rg   rh   s   @r=   rj   rj   c   s~    U "!!)-     	 
     "#   2uS#X  r`   rj   c                   Z   ^  \ rS rSrSr   S	S\S\4U 4S jjjrS\\\4   4S jrSr	U =r
$ )

ConvPosEnc   zlConvolutional Position Encoding.
Note: This module is similar to the conditional position encoding in CPVT.
rE   r   c                 r   > X4S.n[         TU ]  5         [        R                  " XUSUS-  4SU0UD6U l        g )Nr   r   r"   r&   )r'   r(   r-   r2   rw   )r5   rE   r   r    r!   r6   r<   s         r=   r(   ConvPosEnc.__init__   s<     /IIc1adE3E"E	r`   r?   c                 X   UR                   u  p4nUu  pg[        USXg-  -   :H  S5        US S 2S S24   US S 2SS 24   pU	R                  SS5      R                  X5Xg5      n
U R	                  U
5      U
-   nUR                  S5      R                  SS5      n[        R                  " X4SS9nU$ )Nr   rA   r"   rD   )rF   r   rG   viewrw   flattenrI   rL   )r5   r;   r?   rQ   rR   rS   rT   rU   	cls_token
img_tokensfeats              r=   r^   ConvPosEnc.forward   s    ''aQY# !"!RaR%!AqrE(: ##Aq)..qQ:IIdOd"IIaL""1a( IIyn!,r`   )rw   )rq   NN)ra   rb   rc   rd   re   r*   r(   r   r^   rf   rg   rh   s   @r=   r   r      sL     	F	F 	F 	FuS#X  r`   r   c                      ^  \ rS rSrSrSSSSS\R                  \R                  SSSS4S\S\S	\	S
\
S\	S\	S\	S\\R                     S\\R                     S\\   S\\   4U 4S jjjrS\\\4   4S jrSrU =r$ )SerialBlock   z{Serial block class.
Note: In this implementation, each serial block only contains a conv-attention and a FFN (MLP) module.       @Fr   NrE   r   	mlp_ratiorl   rn   rm   	drop_path	act_layer
norm_layer
shared_cpero   c           	      6  > XS.n[         TU ]  5         Xl        U	" U40 UD6U l        [	        U4UUUUUS.UD6U l        US:  a  [        U5      O[        R                  " 5       U l	        U	" U40 UD6U l
        [        X-  5      n[        SUUUUS.UD6U l        g )Nr   r   rl   rm   rn   ro   r   in_featureshidden_featuresr   drop )r'   r(   cpenorm1rj   factoratt_crper   r-   Identityr   norm2r*   r   mlp)r5   rE   r   r   rl   rn   rm   r   r   r   r   ro   r    r!   r6   mlp_hidden_dimr<   s                   r=   r(   SerialBlock.__init__   s      / *r*
5
#
 
 1:B),BKKM  *r*
S_- 
*	

 
r`   r?   c                     U R                  X5      nU R                  U5      nU R                  X25      nXR                  U5      -   nU R	                  U5      nU R                  U5      nXR                  U5      -   nU$ N)r   r   r   r   r   r   )r5   r;   r?   curs       r=   r^   SerialBlock.forward   sp    HHQjjm!!#,s## jjmhhsms##r`   )r   r   r   r   r   r   )ra   rb   rc   rd   re   r-   GELUr   r*   r   r   r   Moduler   r   r(   r   r^   rf   rg   rh   s   @r=   r   r      s    s  ""!!!)+*,,,(,)-+
+
 +
 	+

 +
 +
 +
 +
 BII+
 RYY+
 !+
 "#+
 +
ZuS#X  r`   r   c                   V  ^  \ rS rSrSrSSSSS\R                  \R                  SSS4
S\\	   S\	S\\
   S	\S
\
S\
S\
S\\R                     S\\R                     S\\\      4U 4S jjjrS\
S\\	\	4   4S jrS\
S\\	\	4   4S jrS\
S\\	\	4   4S jrS\\\	\	4      4S jrSrU =r$ )ParallelBlock   zParallel block class. NFr   dimsr   
mlp_ratiosrl   rn   rm   r   r   r   shared_crpesc           	        > XS.n[         TU ]  5         Uc  / nU	" US   40 UD6U l        U	" US   40 UD6U l        U	" US   40 UD6U l        [        US   4UUUUU
S   S.UD6U l        [        US   4UUUUU
S   S.UD6U l        [        US   4UUUUU
S   S.UD6U l        US:  a  [        U5      O[        R                  " 5       U l        U	" US   40 UD6U l        U	" US   40 UD6U l        U	" US   40 UD6U l        US   US   s=:X  a
  US   :X  d   e   eUS   US   s=:X  a
  US   :X  d   e   e[!        US   US   -  5      n[#        SUS   UUUS.UD6=U l        =U l        U l        g )	Nr   r   r"   rq   r   r   r   r   )r'   r(   norm12norm13norm14rj   factoratt_crpe2factoratt_crpe3factoratt_crpe4r   r-   r   r   norm22norm23norm24r*   r   mlp2mlp3mlp4)r5   r   r   r   rl   rn   rm   r   r   r   r   r    r!   r6   r   r<   s                  r=   r(   ParallelBlock.__init__   s    /J !a/B/ a/B/ a/B/6G 
$Q 
  
  7G 
$Q 
  
  7G 
$Q 
  
 1:B),BKKM !a/B/ a/B/ a/B/Aw$q',T!W,,,,,!}
1>A>>>>>T!Wz!}45,/ -
Q*	-

 -
 	
	 	
DI	r`   factorr?   c                 "    U R                  XUS9$ )zFeature map up-sampling. scale_factorr?   interpolater5   r;   r   r?   s       r=   upsampleParallelBlock.upsample@  s    TBBr`   c                 *    U R                  USU-  US9$ )zFeature map down-sampling.       ?r   r   r   s       r=   
downsampleParallelBlock.downsampleD  s    F
FFr`   r   c                 p   UR                   u  pEnUu  px[        USXx-  -   :H  S5        USS2SS2SS24   n	USS2SS2SS24   n
U
R                  SS5      R                  XFXx5      n
[        R
                  " U
USSSS9n
U
R                  XFS5      R                  SS5      n
[        R                  " X4SS	9nU$ )
zFeature map interpolation. r   rA   Nr"   Fbilinear)r   recompute_scale_factormodealign_cornersrB   rD   )rF   r   rG   rH   rM   r   rI   rL   )r5   r;   r   r?   rQ   rR   rS   rT   rU   r   r   outs               r=   r   ParallelBlock.interpolateH  s    ''aQY#a!QhK	q!"ax[
))!Q/77aC
]]%#(

  ''b1;;AqA
ii/Q7
r`   sizesc                 |   Uu  pgpU R                  U5      n
U R                  U5      nU R                  U5      nU R                  XS9n
U R	                  XS9nU R                  XS9nU R                  USUS9nU R                  USU	S9nU R                  USU	S9nU R                  U
SUS9nU R                  USUS9nU R                  U
SUS9nX-   U-   n
X-   U-   nUU-   U-   nX R                  U
5      -   nX0R                  U5      -   nX@R                  U5      -   nU R                  U5      n
U R                  U5      nU R                  U5      nU R                  U
5      n
U R                  U5      nU R                  U5      nX R                  U
5      -   nX0R                  U5      -   nX@R                  U5      -   nXX44$ )Nr}   g       @)r   r?   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r5   x1x2x3x4r   _S2S3S4cur2cur3cur4upsample3_2upsample4_3upsample4_2downsample2_3downsample3_4downsample2_4s                      r=   r^   ParallelBlock.forward_  s   r{{2{{2{{2##D#2##D#2##D#2mmD"m=mmD"m=mmD"m=RbARbARbA!K/!M1m#m3..&&..&&..&& {{2{{2{{2yyyyyy..&&..&&..&&r~r`   )r   r   r   r   r   r   r   r   r   r   r   r   r   )ra   rb   rc   rd   re   r-   r   r   r   r*   r   r   r   r   r   r   r(   r   r   r   r   r^   rf   rg   rh   s   @r=   r   r      s=   !
 '+"!!!)+*,,,04C
s)C
 C
 U	C

 C
 C
 C
 C
 BIIC
 RYYC
 #49-C
 C
JC% CuS#X CGE GsCx G5 c3h . T%S/-B    r`   r   c            '         ^  \ rS rSrSrSSSSSSS	S
SSSSSS\SSSSSS4S\S\S\S\S\\\\\4   S\\\\\4   S\S\S\\\\\4   S\	S\S\S\S\S\
\R                     S \	S!\\\      S"\\   S#\4&U 4S$ jjjrS% r\R(                  R*                  S& 5       r\R(                  R*                  S2S' j5       r\R(                  R*                  S3S( j5       r\R(                  R*                  S)\R                  4S* j5       rS4S\S#\\   4S+ jjrS, rS3S-\\R:                  \\R:                     4   S.\	4S/ jjrS)\R:                  4S0 jrS1r U =r!$ )5r   i  zCoaT class.       rq     @      @     rq   r|      rq   r   r   )r|   r|   r|   r|   Tr   FNtokenimg_size
patch_sizein_chansnum_classes
embed_dimsserial_depthsparallel_depthr   r   rl   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   return_interm_layersout_featurescrpe_windowglobal_poolc                 (  > [         TU ]  5         UUS.nUS;   d   eU=(       d    SSSS.nUU l        UU l        XPl        US   =U l        U l        X@l        X0l        UU l	        [        U5      n[        SXUUS   [        R                  S.UD6U l        [        SU Vs/ s H  nUS	-  PM
     snSUS   US
   [        R                  S.UD6U l        [        SU Vs/ s H  nUS-  PM
     snSUS
   US   [        R                  S.UD6U l        [        SU Vs/ s H  nUS-  PM
     snSUS   US   [        R                  S.UD6U l        [        R$                  " [&        R(                  " S
S
US   40 UD65      U l        [        R$                  " [&        R(                  " S
S
US
   40 UD65      U l        [        R$                  " [&        R(                  " S
S
US   40 UD65      U l        [        R$                  " [&        R(                  " S
S
US   40 UD65      U l        [3        SUS   SS.UD6U l        [3        SUS
   SS.UD6U l        [3        SUS   SS.UD6U l        [3        SUS   SS.UD6U l        [=        SUS   U-  UUS.UD6U l        [=        SUS
   U-  UUS.UD6U l         [=        SUS   U-  UUS.UD6U l!        [=        SUS   U-  UUS.UD6U l"        Un[G        UU
UUUUS9n[        RH                  " [K        US   5       Vs/ s H/  n[M        SUS   U	S   U R4                  U R>                  S.UDUD6PM1     sn5      U l'        [        RH                  " [K        US
   5       Vs/ s H/  n[M        SUS
   U	S
   U R6                  U R@                  S.UDUD6PM1     sn5      U l(        [        RH                  " [K        US   5       Vs/ s H/  n[M        SUS   U	S   U R8                  U RB                  S.UDUD6PM1     sn5      U l)        [        RH                  " [K        US   5       Vs/ s H/  n[M        SUS   U	S   U R:                  U RD                  S.UDUD6PM1     sn5      U l*        Xpl+        U RV                  S:  aq  [        RH                  " [K        U5       Vs/ s H@  n[Y        SUU	U R>                  U R@                  U RB                  U RD                  4S.UDUD6PMB     sn5      U l-        OS U l-        U R                  (       Gde  U RZ                  b#  U" US
   40 UD6U l.        U" US   40 UD6U l/        OS =U l.        U l/        U" US   40 UD6U l0        U RV                  S:  a  US
   US   s=:X  a
  US   :X  d   e   e[&        R                  Rb                  " SSS
S
S.UD6U l2        [        Rf                  " U5      U l4        US:  a"  [        Rj                  " U R
                  U40 UD6O[        Rl                  " 5       U l7        OdS U l2        [        Rf                  " U5      U l4        US:  a"  [        Rj                  " U R
                  U40 UD6O[        Rl                  " 5       U l7        [q        U R*                  SS9  [q        U R,                  SS9  [q        U R.                  SS9  [q        U R0                  SS9  U Rs                  U Rt                  5        g s  snf s  snf s  snf s  snf s  snf s  snf s  snf s  snf )Nr   r   avgr"   rq   )rq         rB   r   )r   r   r   	embed_dimr   r|   r   r   r   )rE   r   )r   r   r   )r   rl   rn   rm   r   r   )rE   r   r   ro   )r   r   r   )in_channelsout_channelsr#   {Gz?stdr   );r'   r(   r  r  r   num_featureshead_hidden_sizer   r   r	  r   r   r-   r   patch_embed1patch_embed2patch_embed3patch_embed4	ParameterrI   zeros
cls_token1
cls_token2
cls_token3
cls_token4r   cpe1cpe2cpe3cpe4r   crpe1crpe2crpe3crpe4r+   r.   ranger   serial_blocks1serial_blocks2serial_blocks3serial_blocks4r  r   parallel_blocksr   norm3norm4Conv1d	aggregaterv   	head_droprt   r   headr   apply_init_weights)r5   r   r   r   r   r   r   r  r   r   rl   r  r  r  r  r   r  r  r  r	  r    r!   r6   r;   dprskwargsr   r<   s                              r=   r(   CoaT.__init__  s{   0 	/....!7qQ%7$8!($4>rNBD1& & X&& Dx mD@BD ' D&./ha1fh/A
ST mD@BD ' D&./ha1fh/A
ST mD@BD ' D'/0x!a2gx0QTU mD@BD
 ,,u{{1aA'M"'MN,,u{{1aA'M"'MN,,u{{1aA'M"'MN,,u{{1aA'M"'MN <:a=A<<	<:a=A<<	<:a=A<<	<:a=A<<	 #vJqMY,FR[dovsuv
"vJqMY,FR[dovsuv
"vJqMY,FR[dovsuv
"vJqMY,FR[dovsuv
$$!
 !mm =+,	-. -  qM$Q-99 JJ	
   -	-. 

 !mm =+,	-. -  qM$Q-99 JJ	
   -	-. 

 !mm =+,	-. -  qM$Q-99 JJ	
   -	-. 

 !mm =+,	-. -  qM$Q-99 JJ	
   -	-. 

 -"#%== ~.20 /A  #)"&**djj$**djj!Q 	
  /20 	$D  $(D  (((##/'
1<<
'
1<<
*..
TZ#JqM8R8DJ""Q&!!}
1FAFFFFF!&!dQQ\]!dac!d!#I!6OZ]^BIId&7&7KKdfdododq	 "&!#I!6OZ]^BIId&7&7KKdfdododq	 	doo3/doo3/doo3/doo3/

4%%&G 0 0 1@	-.	-.	-.	-.20s1   [,
[1
[6
6[;56\ 6\96\
.A\c                    [        U[        R                  5      (       am  [        UR                  SS9  [        U[        R                  5      (       a9  UR
                  b+  [        R                  R                  UR
                  S5        g g g [        U[        R                  5      (       aU  [        R                  R                  UR
                  S5        [        R                  R                  UR                  S5        g g )Nr  r  r   r   )	r)   r-   rt   r   weightrr   init	constant_r   )r5   ms     r=   r6  CoaT._init_weights3  s    a##!((,!RYY''AFF,>!!!&&!, -?'2<<((GGaffa(GGahh, )r`   c                 
    1 Sk$ )N>   r  r  r  r   r   r5   s    r=   no_weight_decayCoaT.no_weight_decay<  s    GGr`   c                      U(       a   S5       eg )Nz$gradient checkpointing not supportedr   )r5   enables     r=   set_grad_checkpointingCoaT.set_grad_checkpointing@  s    AAAz6r`   c                 ,    [        SSSSSSSSS	S
/S9	nU$ )Nz#^cls_token1|patch_embed1|crpe1|cpe1z^serial_blocks1\.(\d+)z#^cls_token2|patch_embed2|crpe2|cpe2z^serial_blocks2\.(\d+)z#^cls_token3|patch_embed3|crpe3|cpe3z^serial_blocks3\.(\d+)z#^cls_token4|patch_embed4|crpe4|cpe4z^serial_blocks4\.(\d+))z^parallel_blocks\.(\d+)N)z^norm|aggregate)i )	stem1r*  stem2r+  stem3r,  stem4r-  r.  )r+   )r5   coarsematchers      r=   group_matcherCoaT.group_matcherD  s6    848484842.
 r`   returnc                     U R                   $ r   )r4  rA  s    r=   get_classifierCoaT.get_classifierV  s    yyr`   c                     Xl         Ub  US;   d   eX l        US:  a'  [        R                  " U R                  U5      U l        g [        R
                  " 5       U l        g )Nr  r   )r   r	  r-   rt   r  r   r4  )r5   r   r	  s      r=   reset_classifierCoaT.reset_classifierZ  sQ    &""2222*ALqBIId//=	VXVaVaVc	r`   c                 X
   UR                   S   nU R                  U5      nU R                  R                  u  pE[        X0R                  5      nU R
                   H  nU" X4U4S9nM     [        U5      R                  X$US5      R                  SSSS5      R                  5       nU R                  U5      nU R                  R                  u  p[        XR                  5      nU R                   H  nU" XU
4S9nM     [        U5      R                  X)U
S5      R                  SSSS5      R                  5       nU R                  U5      nU R                  R                  u  p[        XR                  5      nU R                   H  nU" XU4S9nM     [        U5      R                  X-US5      R                  SSSS5      R                  5       nU R!                  U5      nU R                   R                  u  nn[        UU R"                  5      nU R$                   H  nU" UUU4S9nM     [        U5      R                  UUUS5      R                  SSSS5      R                  5       nU R&                  c  [(        R*                  R-                  5       (       di  U R.                  (       aX  0 nSU R0                  ;   a  UUS'   SU R0                  ;   a  UUS'   S	U R0                  ;   a  UUS	'   S
U R0                  ;   a  UUS
'   U$ U R3                  U5      nU$ U R&                   HR  nU R5                  XU
45      U R7                  XU45      U R9                  UUU45      npU" X8UUXE4X4X4UU4/S9u  p8nnMT     [(        R*                  R-                  5       (       Gd[  U R.                  (       GaI  0 nSU R0                  ;   aA  [        U5      R                  X$US5      R                  SSSS5      R                  5       nUUS'   SU R0                  ;   aA  [        U5      R                  X)U
S5      R                  SSSS5      R                  5       nUUS'   S	U R0                  ;   aA  [        U5      R                  X-US5      R                  SSSS5      R                  5       nUUS	'   S
U R0                  ;   aB  [        U5      R                  UUUS5      R                  SSSS5      R                  5       nUUS
'   U$ U R;                  U5      nU R=                  U5      nU R3                  U5      nXU/$ )Nr   r}   rB   rq   r   r"   x1_noclsx2_noclsx3_noclsx4_nocls)r   )rF   r  	grid_size
insert_clsr  r*  
remove_clsrH   r~   
contiguousr  r  r+  r  r  r,  r  r   r-  r.  rI   jitis_scriptingr  r  r0  r"  r#  r$  r   r/  )r5   x0rQ   r   H1W1blkrY  r   H2W2rZ  r   H3W3r[  r   H4W4r\  feat_outs                        r=   forward_featuresCoaT.forward_featuresa  s   HHQK r""",,OO,&&CR2h'B 'b>))!R8@@Aq!LWWY x("",,OO,&&CR2h'B 'b>))!R8@@Aq!LWWY x("",,OO,&&CR2h'B 'b>))!R8@@Aq!LWWY x("",,BDOO,&&CRr2h'B 'b>))!RR8@@Aq!LWWY '99))++0I0I!2!22+3HZ(!2!22+3HZ(!2!22+3HZ(!2!22+3HZ( ZZ^	 ''C2Bx0$))BR2I499UWZ\^`YaKbB R2(RHWY[]V^7_`NBB ( yy%%''D,E,E,EHT...%b>11!R@HHAqRST__a'/$T...%b>11!R@HHAqRST__a'/$T...%b>11!R@HHAqRST__a'/$T...%b>11!RR@HHAqRST__a'/$OBBBBBBB<r`   x_feat
pre_logitsc           
      F   [        U[        5      (       a  U R                  c   eU R                  S:X  a=  [        R
                  " U Vs/ s H  o3S S 2SS 24   R                  SSS9PM     snSS9nO,[        R                  " U Vs/ s H  o3S S 2S4   PM     snSS9nU R                  U5      R                  SS9nO1U R                  S:X  a  US S 2SS 24   R                  SS9OUS S 2S4   nU R                  U5      nU(       a  U$ U R                  U5      $ s  snf s  snf )Nr  r   T)rE   keepdimrD   r   )r)   listr2  r	  rI   rL   meanstacksqueezer3  r4  )r5   rp  rq  xlr;   s        r=   forward_headCoaT.forward_head  s   fd##>>---5(IIVTVr!QR%y~~!T~BVTZ[\KKF ;FbAqDF ;Cq!))a)0A-1-=-=-Fq!"u""q")FSTVWSWLANN1q0DIIaL0 U ;s   	"DDc                     [         R                  R                  5       (       d"  U R                  (       a  U R	                  U5      $ U R	                  U5      nU R                  U5      nU$ r   )rI   ra  rb  r  rn  ry  )r5   r;   rp  s      r=   r^   CoaT.forward  sU    yy%%''D,E,E((++ **1-F!!&)AHr`   )$r2  r  r  r  r   r!  r"  r#  r$  r%  r&  r'  r(  r   r	  r4  r3  r  r   r   r/  r0  r   r  r  r.  r  r  r  r  r  r  r*  r+  r,  r-  )TFr   )"ra   rb   rc   rd   re   r   r*   r   r   r   r   r-   r   r   r   strr+   r(   r6  rI   ra  ignorerB  rF  rO  rS  rV  rn  r   Tensorry  r^   rf   rg   rh   s   @r=   r   r     sO      #4G7C"#<H!!$&$&$&*3).04*.&-m'm' m' 	m'
 m' c3S01m' !c3!34m'  m' m' eUE589m' m' m' "m' "m' "m'  RYY!m'" #'#m'$ #49-%m'& "$'m'( )m' m'^- YYH H YYB B YY " YY		  dC dhsm dO b15tELL7I)I#J 1X\ 1ELL  r`   c                 r    UR                  U R                  S   SS5      n[        R                  " X 4SS9n U $ )zInsert CLS token. r   rB   r   rD   )expandrF   rI   rL   )r;   r   
cls_tokenss      r=   r^  r^    s5    !!!''!*b"5J		:/q)AHr`   c                     U SS2SS2SS24   $ )zRemove CLS token. Nr   r   )r;   s    r=   r_  r_    s    QAX;r`   c                     0 nU R                  SU 5      n U R                  5        H  u  p4UR                  S5      (       d  UR                  S5      (       a  [        USS 5      b  UR                  S5      (       a  [        USS 5      bl  UR                  S5      (       a  [        USS 5      bH  UR                  S5      (       a  [        USS 5      b$  UR                  S5      (       a  [        USS 5      c  M  XBU'   M     U$ )Nmodelr   r   r/  r0  r2  r4  )getr1   
startswithgetattr)
state_dictr  out_dictr   rP   s        r=   checkpoint_filter_fnr    s    H4J  "<<  g&&75'4+H+Pg&&75'4+H+Pg&&75'4+H+Pk**wuk4/P/Xf%%'%*F*N # Or`   c                 z    UR                  SS 5      (       a  [        S5      e[        [        U U4S[        0UD6nU$ )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.pretrained_filter_fn)r  RuntimeErrorr   r   r  )variant
pretraineddefault_cfgkwargsr  s        r=   _create_coatr    sM    zz/4((YZZ  2	
 E Lr`   c                 4    U SSS SSS[         [        SSSS	.UE$ )
Nr   )rq   r   r   g?bicubicTzpatch_embed1.projr4  z
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizeru  r  
first_conv
classifierlicenser	   )r  r  s     r=   	_cfg_coatr    s5    =t%.B)  r`   ztimm/)	hf_hub_id)rq     r  r   squash)r  r  r  	crop_mode)zcoat_tiny.in1kzcoat_mini.in1kzcoat_small.in1kzcoat_lite_tiny.in1kzcoat_lite_mini.in1kzcoat_lite_small.in1kzcoat_lite_medium.in1kzcoat_lite_medium_384.in1krQ  c           	      T    [        S/ SQ/ SQSS9n[        SSU 0[        U40 UD6D6nU$ )Nr|   )r   r   r   r   r"   r"   r"   r"   r   r   r   r   r  r  )	coat_tinyr+   r  r  r  	model_cfgr  s       r=   r  r    :    !5\bceIYYtI?XQW?XYELr`   c           	      T    [        S/ SQ/ SQSS9n[        SSU 0[        U40 UD6D6nU$ )Nr|   )r      r  r  r  r   r  r  )	coat_minir  r  s       r=   r  r    r  r`   c           	      \    [        SS/ SQ/ SQSS.UD6n[        SSU 0[        U40 UD6D6nU$ )	Nr|   )r   r   r   r   r  r   r  r  r   )
coat_smallr  r  s       r=   r  r    sI     o!5\bcogmoIZ*ZY@YRX@YZELr`   c           	      X    [        S/ SQ/ SQ/ SQS9n[        SSU 0[        U40 UD6D6nU$ )Nr|   )r   r      r   r  r   r   r|   r|   r   r   r   r   r  )coat_lite_tinyr  r  s       r=   r  r  '  :    !4L]ikI^j^DD]V\D]^ELr`   c           	      X    [        S/ SQ/ SQ/ SQS9n[        SSU 0[        U40 UD6D6nU$ )Nr|   r   r  r  r  r  )coat_lite_minir  r  s       r=   r  r  /  r  r`   c           	      X    [        S/ SQ/ SQ/ SQS9n[        SSU 0[        U40 UD6D6nU$ )Nr|   r   r   r  r  r  )coat_lite_smallr  r  s       r=   r  r  7  s:    !4L]ikI_z_T)E^W]E^_ELr`   c           	      R    [        S/ SQ/ SQS9n[        SSU 0[        U40 UD6D6nU$ )Nr|   r   r  r   r   rq   r   
   r   )r   r   r   r  )coat_lite_mediumr  r  s       r=   r  r  ?  s7    !5]TI`
`d9F_X^F_`ELr`   c           	      T    [        SS/ SQ/ SQS9n[        SSU 0[        U40 UD6D6nU$ )Nr  r|   r  r  )r   r   r   r   r  )coat_lite_medium_384r  r  s       r=   r  r  G  s:    /CS`bIdJd$yJc\bJcdELr`   )FN)rA   r}  )4re   typingr   r   r   r   r   r   rI   torch.nnr-   torch.nn.functional
functionalrM   	timm.datar
   r   timm.layersr   r   r   r   r   r   r   _builderr   	_registryr   r   __all__r   r   rj   r   r   r   r   r^  r_  r  r  r  default_cfgsr  r  r  r  r  r  r  r  r   r`   r=   <module>r     s   ; :     A _ _ _ * <(HBII HV2bii 2j D<")) <~FBII FRE299 EP

 	 %'2'2 73$w7$w7%8&9!* 3("&  T   T   d   $   $   4   D     r`   