
    RЦiP                        S r SSKJrJrJrJr  SSKrSSKJr  SSK	J
r
Jr  SSKJrJrJrJrJrJrJrJr  SSKJr  SSKJr  SS	KJrJr  S
/r " S S\R:                  5      r " S S\R:                  5      r " S S\R:                  5      r  " S S
\R:                  5      r!SS jr"SS jr#\" \#" SS9\#" SS9S.5      r$\SS\!4S jj5       r%\SS\!4S jj5       r&g)zVisformer

Paper: Visformer: The Vision-friendly Transformer - https://arxiv.org/abs/2104.12533

From original at https://github.com/danczs/Visformer

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )OptionalUnionTypeAnyNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)	to_2tupletrunc_normal_DropPathcalculate_drop_path_rates
PatchEmbedLayerNorm2dcreate_classifieruse_fused_attn   )build_model_with_cfg)checkpoint_seq)register_modelgenerate_default_cfgs	Visformerc                      ^  \ rS rSrSS\R
                  SSSSS4S\S\\   S\\   S	\\R                     S
\
S\S\4U 4S jjjrS rSrU =r$ )
SpatialMlp   N           Fin_featureshidden_featuresout_features	act_layerdropgroupspatial_convc
                   > XS.n
[         TU ]  5         U=(       d    UnU=(       d    Un[        U5      nXl        X0l        Xpl        U R
                  (       a  US:  a	  US-  S-  nOUS-  nX l        X`l        [        R                  " XS4SSSS.U
D6U l
        U" 5       U l        [        R                  " US   5      U l        U R
                  (       a9  [        R                  " X"S	4SSU R                  SS
.U
D6U l        U" 5       U l        OS U l        S U l        [        R                  " X#S4SSSS.U
D6U l        [        R                  " US   5      U l        g )Ndevicedtype         r   r   Fstridepaddingbias   )r,   r-   groupsr.   )super__init__r
   r   r   r#   r   r"   nnConv2dconv1act1Dropoutdrop1conv2act2conv3drop3)selfr   r   r   r    r!   r"   r#   r&   r'   dd
drop_probs	__class__s               T/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/visformer.pyr2   SpatialMlp.__init__   sF    /#2{)8[t_
&((qy"-/Q"6"-/.
YY{QfqRSZ_fcef
K	ZZ
1.
!o<=qQUQ[Q[bgokmoDJ!DIDJDIYYagST[`gdfg
ZZ
1.
    c                    U R                  U5      nU R                  U5      nU R                  U5      nU R                  b"  U R                  U5      nU R	                  U5      nU R                  U5      nU R                  U5      nU$ N)r5   r6   r8   r9   r:   r;   r<   r=   xs     rA   forwardSpatialMlp.forwardB   sl    JJqMIIaLJJqM::!

1A		!AJJqMJJqMrC   )r6   r:   r5   r9   r;   r8   r<   r"   r   r   r   r#   )__name__
__module____qualname____firstlineno__r3   GELUintr   r   Modulefloatboolr2   rH   __static_attributes____classcell__r@   s   @rA   r   r      s     .2*.)+!&'/'/ &c]'/ #3-	'/
 BII'/ '/ '/ '/ '/R	 	rC   r   c                      ^  \ rS rSr% \R
                  R                  \   \S'         SS\	S\	S\
S\
S\
4
U 4S jjjrS	 rS
rU =r$ )	AttentionN   
fused_attndim	num_headshead_dim_ratio	attn_drop	proj_dropc                   > XgS.n[         T
U ]  5         Xl        X l        [	        X-  U-  5      n	Xl        U	S-  U l        [        SS9U l        [        R                  " XU-  S-  S4SSSS	.UD6U l        [        R                  " U5      U l        [        R                  " U R
                  U R                  -  US4SSSS	.UD6U l        [        R                  " U5      U l        g )
Nr%   g      T)experimentalr/   r   r   Fr+   )r1   r2   rZ   r[   roundhead_dimscaler   rY   r3   r4   qkvr7   r]   projr^   )r=   rZ   r[   r\   r]   r^   r&   r'   r>   rb   r@   s             rA   r2   Attention.__init__Q   s     /")N:; %
(d;99SY"6":AeaQRY^ebdeI.IIdmmdnn<c1lQXY`elikl	I.rC   c                    UR                   u  p#pEU R                  U5      R                  USU R                  U R                  S5      R                  SSSSS5      nUR                  S5      u  pgnU R                  (       a|  [        R                  R                  R                  UR                  5       UR                  5       UR                  5       U R                  (       a  U R                  R                  OSS9nOEXgR!                  S	S5      -  U R"                  -  n	U	R%                  SS
9n	U R                  U	5      n	X-  nUR                  SSSS5      R                  USXE5      nU R'                  U5      nU R)                  U5      nU$ )Nr/   r   r   r(      r   )	dropout_p)rZ   )shaperd   reshaper[   rb   permuteunbindrY   torchr3   
functionalscaled_dot_product_attention
contiguoustrainingr]   p	transposerc   softmaxre   r^   )
r=   rG   BCHWqkvattns
             rA   rH   Attention.forwardi   s;   WW
aHHQK1dnndmmRHPPQRTUWXZ[]^_((1+a??##@@.2mm$..** A A
 B++tzz9D<<B<'D>>$'DAIIaAq!))!R6IIaLNN1rC   )	r]   rZ   rY   rb   r[   re   r^   rd   rc   )r         ?r   r   NN)rJ   rK   rL   rM   rp   jitFinalrR   __annotations__rO   rQ   r2   rH   rS   rT   rU   s   @rA   rW   rW   N   sn    		%%
 $&!!// / "	/
 / / /0 rC   rW   c                      ^  \ rS rSrSSSSS\R
                  \SSSSS4S\S	\S
\S\S\S\S\S\	\R                     S\	\R                     S\S\S\4U 4S jjjrS rSrU =r$ )Block   r         @r   r   FNrZ   r[   r\   	mlp_ratior^   r]   	drop_pathr    
norm_layerr"   attn_disabledr#   c           
      `  > XS.n[         TU ]  5         Xl        US:  a  [        U5      O[        R
                  " 5       U l        U(       a  S U l        S U l        O$U	" U40 UD6U l        [        U4UUUUS.UD6U l        U	" U40 UD6U l
        [        SU[        X-  5      UUU
US.UD6U l        g )Nr%   r   )r[   r\   r]   r^   )r   r   r    r!   r"   r#    )r1   r2   r#   r   r3   Identityr   norm1r   rW   norm2r   rO   mlp)r=   rZ   r[   r\   r   r^   r]   r   r    r   r"   r   r#   r&   r'   r>   r@   s                   rA   r2   Block.__init__   s    " /(09B),BKKMDJDI#C.2.DJ!#-## DI  *r*
 
0%
 
rC   c                     U R                   b1  XR                  U R                  U R                  U5      5      5      -   nXR                  U R                  U R	                  U5      5      5      -   nU$ rE   )r   r   r   r   r   rF   s     rA   rH   Block.forward   sU    99 NN499TZZ]#;<<Atxx

1677rC   )r   r   r   r   r   r#   )rJ   rK   rL   rM   r3   rN   r   rO   rQ   r   rP   rR   r2   rH   rS   rT   rU   s   @rA   r   r      s    
 %'!!!!)+*5"'!&,
,
 ,
 "	,

 ,
 ,
 ,
 ,
 BII,
 RYY,
 ,
  ,
 ,
 ,
\ rC   r   c            1         ^  \ rS rSrSSSSSSSS	S
SSSSS\SSSSSSSSSS4S\S\S\S\S\\   S\S\\\4   S\S\	S\	S\	S\	S\	S\	S \
\R                     S!\S"\S#\S$\S%\S&\S'\S(\\
\R                        4.U 4S) jjjrS* r\R$                  R&                  S5S+ j5       r\R$                  R&                  S6S, j5       r\R$                  R&                  S-\R                  4S. j5       rS7S\S&\4S/ jjrS0 rS5S1\4S2 jjrS3 rS4rU =r$ )8r            r/              r*   r   r   111TFr   avgNimg_size
patch_sizein_chansnum_classesinit_channels	embed_dimdepthr[   r   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   
attn_stageuse_pos_embedr#   vit_stemr"   global_pool	conv_init
embed_normc                   > [         TU ]  5         UUS.n[        U5      nX@l        X0l        X`l        XPl        Xl        UU l        UU l	        [        U[        [        45      (       a!  Uu  U l        U l        U l        [!        U5      nO1US-  =U l        U l        XpR                  -
  U R                  -
  U l        UU l        SU l        ['        X5      nU R                  (       a5  S U l        [+        SUUUUUSS.UD6U l        U Vs/ s H  nUU-  PM
     nnGOU R                  c=  S U l        [+        SUUS-  UUS-  USS.UD6U l        U Vs/ s H  nUUS-  -  PM     nnO[.        R0                  " [.        R2                  " X0R                  S4SSSS.UD6[.        R4                  " U R                  40 UD6[.        R6                  " SS	95      U l        U Vs/ s H  nUS-  PM
     nn[+        SUUS
-  U R                  US-  USS.UD6U l        U Vs/ s H  nUUS
-  -  PM     nnU R"                  (       a  U R                  (       a5  [.        R8                  " [:        R<                  " SU/UQ70 UD65      U l        O7[.        R8                  " [:        R<                  " SUS-  /UQ70 UD65      U l        [.        R@                  " US9U l!        OS U l        [.        R0                  " [E        U R                  5       Vs/ s H,  n[G        SUS-  USU	UUUU   UUUS   S:H  US   S:H  S.UD6PM.     sn6 U l$        U R                  (       d  [+        SUUS-  US-  UUSS.UD6U l%        U Vs/ s H  nUUS-  -  PM     nnU R"                  (       a5  [.        R8                  " [:        R<                  " SU/UQ70 UD65      U l&        OS U l&        OS U l%        [.        R0                  " [E        U R                  U R                  U R                  -   5       Vs/ s H)  n[G        SUUSU	UUUU   UUUS   S:H  US   S:H  S.UD6PM+     sn6 U l'        U R                  (       d  [+        SUUS-  UUS-  USS.UD6U l(        U Vs/ s H  nUUS-  -  PM     nnU R"                  (       a8  [.        R8                  " [:        R<                  " SUS-  /UQ70 UD65      U l)        OS U l)        OS U l(        [.        R0                  " [E        U R                  U R                  -   U5       Vs/ s H,  n[G        SUS-  USU	UUUU   UUUS   S:H  US   S:H  S.UD6PM.     sn6 U l*        U R                  (       a  UOUS-  =U l+        U l,        U" U RV                  40 UD6U l-        []        U RV                  U R                  UUUS9u  nnUU l/        [.        R@                  " U
5      U l0        UU l1        U R"                  (       aM  [e        U R>                  SS9  U R                  (       d(  [e        U RL                  SS9  [e        U RR                  SS9  U Rg                  U Rh                  5        g s  snf s  snf s  snf s  snf s  snf s  snf s  snf s  snf s  snf )Nr%   r/   F)r   r   r   r   r   flattenr(      r+   T)inplaceri   r   )ru   g      ?r   01)rZ   r[   r\   r   r^   r]   r   r   r"   r   r#   r   r   	pool_typer&   r'   {Gz?stdr   )5r1   r2   r
   r   r   r   r   r   r   r   
isinstancelisttuple
stage_num1
stage_num2
stage_num3sumr   grad_checkpointingr   stemr   patch_embed1r3   
Sequentialr4   BatchNorm2dReLU	Parameterrp   zeros
pos_embed1r7   pos_dropranger   stage1patch_embed2
pos_embed2stage2patch_embed3
pos_embed3stage3num_featureshead_hidden_sizenormr   r   	head_dropheadr   apply_init_weights) r=   r   r   r   r   r   r   r   r[   r   r   r   r   r   r   r   r   r   r#   r   r"   r   r   r   r&   r'   r>   dprrG   ir   r@   s                                   rA   r2   Visformer.__init__   s   8 	/X&& "*  "edE]++@E=DOT_doJE05
:DOdo#oo5GDO*"''>==DI * !!%!#%! !D 2::AZH:H!!) 	$. %%)Q%'1n)!% %! =EEHqA*/2HEMMIIh(:(:AeaQRY^ebdeNN4#5#5<<GGD)	
 -55HqAFH5$. %%)Q!//'1n)!% %! =EEHqA*/2HE}}"$,,u{{1i/Y(/YVX/Y"Z"$,,u{{1il/\X/\Y[/\"]JJ7DM"DOmm 4??+&
 ,  qL#"#((a&%)!}3*1o4  ,&
 & }} * !!%?"a#%! !D 9AA1jAo.HA!!"$,,u{{1i/Y(/YVX/Y"Z"& $Dmm 4??DOODOO,KL&
 M  #"#((a&%)!}3*1o4  M&
 & }} * !!%?"#a-%! !D 9AA1jAo.HA!!"$,,u{{1ik/[H/[XZ/["\"& $Dmm 4??4??:EB&
 C  M#"#((a&%)!}3*1o4  C&
 $ BFIT]`aTaaD1t007B7	 .!
T 'I.	 $//t4==doo48doo48

4%%&q ; F 6 F&
: B&
: B&
s6   YY$Y)Y.>3Y3,Y80Y=?Z 3Zc                 *   [        U[        R                  5      (       aM  [        UR                  SS9  UR
                  b+  [        R                  R                  UR
                  S5        g g [        U[        R                  5      (       a  U R                  (       a*  [        R                  R                  UR                  SSS9  O[        UR                  SS9  UR
                  b+  [        R                  R                  UR
                  S5        g g g )Nr   r   r   fan_outrelu)modenonlinearityr   )r   r3   Linearr   weightr.   init	constant_r4   r   kaiming_normal_)r=   ms     rA   r   Visformer._init_weights  s    a##!((-vv!!!!&&!, "299%%~~''yv'VahhD1vv!!!!&&"- " &rC   c                 2    [        SU(       a  SOSS 4SS/S9$ )Nz^patch_embed1|pos_embed1|stemz^stage(\d+)\.(\d+))z^(?:patch_embed|pos_embed)(\d+))r   )z^norm)i )r   blocks)dict)r=   coarses     rA   group_matcherVisformer.group_matcher  s+    1*0&6KTR:$
 	
rC   c                     Xl         g rE   )r   )r=   enables     rA   set_grad_checkpointing Visformer.set_grad_checkpointing  s    "(rC   returnc                     U R                   $ rE   )r   )r=   s    rA   get_classifierVisformer.get_classifier  s    yyrC   c                 ^   Xl         [        U R                  S5      (       a   U R                  R                  R                  OS n[        U R                  S5      (       a   U R                  R                  R
                  OS n[        U R                  U R                   X#US9u  U l        U l        g )Nr   r   )	r   hasattrr   r   r&   r'   r   r   r   )r=   r   r   r&   r'   s        rA   reset_classifierVisformer.reset_classifier  s    &,3DIIx,H,H!!((d*1$))X*F*F		  &&D&7t//;]b'd#$)rC   c                 "   U R                   b  U R                  U5      nU R                  U5      nU R                  b  U R                  XR                  -   5      nU R                  (       a:  [
        R                  R                  5       (       d  [        U R                  U5      nOU R                  U5      nU R                  b;  U R                  U5      nU R                  b  U R                  XR                  -   5      nU R                  (       a:  [
        R                  R                  5       (       d  [        U R                  U5      nOU R                  U5      nU R                  b;  U R                  U5      nU R                  b  U R                  XR                  -   5      nU R                  (       a:  [
        R                  R                  5       (       d  [        U R                  U5      nOU R                  U5      nU R!                  U5      nU$ rE   )r   r   r   r   r   rp   r   is_scriptingr   r   r   r   r   r   r   r   r   rF   s     rA   forward_featuresVisformer.forward_features  s~   99 		!A a ??&a//12A""599+A+A+C+Ct{{A.AAA (!!!$A*MM!oo"56""599+A+A+C+Ct{{A.AAA (!!!$A*MM!oo"56""599+A+A+C+Ct{{A.AAAIIaLrC   
pre_logitsc                 z    U R                  U5      nU R                  U5      nU(       a  U$ U R                  U5      $ rE   )r   r   r   )r=   rG   r  s      rA   forward_headVisformer.forward_head  s5    QNN1q0DIIaL0rC   c                 J    U R                  U5      nU R                  U5      nU$ rE   )r   r  rF   s     rA   rH   Visformer.forward  s'    !!!$a rC   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   F)T)r   )rJ   rK   rL   rM   r   rO   r   r   r   rQ   r   r3   rP   strrR   r2   r   rp   r   ignorer   r   r   r   r   r  rH   rS   rT   rU   s   @rA   r   r      s      #+- ')!!#%$&$&$&*5#"& %"$#485T'T' T' 	T'
 T' $C=T' T' e$T' T' T' T' !T' "T' "T' "T'  RYY!T'" #T'$  %T'& 'T'( )T'* +T', -T'. /T'0 !bii11T' T'l. YY
 
 YY) ) YY		  dC dc d"H1$ 1
 rC   c                 l    UR                  SS 5      (       a  [        S5      e[        [        X40 UD6nU$ )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretraineddefault_cfgkwargsmodels        rA   _create_visformerr    s5    zz/4((YZZ GJ6JELrC   c                 4    U SSSSSS[         [        SSS	S
.UE$ )Nr   )r/   r   r   )r   r   g?bicubicTzstem.0r   z
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizemeanr   
first_conv
classifierlicenser   )r  r  s     rA   _cfgr!    s5    =v%.Bf  rC   ztimm/)	hf_hub_id)zvisformer_tiny.in1kzvisformer_small.in1kr   c                     [        SSSSSSSS[        R                  S	[        R                  S
9n[        SSU 0[        U40 UD6D6nU$ )Nr      r   ri   ri   r/   r   r   011100Tr   r   r   r[   r   r"   r   r#   r   r   r   r  )visformer_tinyr   r3   r   r  r  r  	model_cfgr  s       rA   r)  r)    sT    CyAQS[\uSW>>#I c:ciIb[aIbcELrC   c                     [        SSSSSSSS[        R                  S	[        R                  S
9n[        SSU 0[        U40 UD6D6nU$ )Nr   r   r%  r*   r   r   r&  r'  Tr(  r  )visformer_smallr*  r+  s       rA   r.  r.    sT    CyAQS[\uSW>>#I dJd$yJc\bJcdELrC   )FN) r  )'__doc__typingr   r   r   r   rp   torch.nnr3   	timm.datar   r	   timm.layersr
   r   r   r   r   r   r   r   _builderr   _manipulater   	_registryr   r   __all__rP   r   rW   r   r   r  r!  default_cfgsr)  r.  r   rC   rA   <module>r:     s    . -   A R  R  R * ' <-3 3l.		 .b3BII 3lj		 jZ		 %'2 73&  )   9  rC   