
    RЦi$a              
       b   S r SSKrSSKJr  SSKJrJrJrJrJ	r	J
r
  SSKrSSKJr  SSKJrJr  SSKJrJrJrJrJrJrJrJrJrJrJrJrJr  SSKJ r   SS	K!J"r"  SS
K#J$r$  SSK%J&r&J'r'  SSK(J)r)J*r*  S/r+ " S S\RX                  5      r- " S S\RX                  5      r. " S S\RX                  5      r/ " S S\RX                  5      r0 " S S\RX                  5      r1S\\2\24   4S jr3\$S\\2\24   S\\2\24   4S j5       r4 " S S\RX                  5      r5 " S S\RX                  5      r6 " S  S\RX                  5      r7S0S! jr8S1S" jr9\*" \9" S#S$9\9" S%S$9\9" S&S$9\9" S'S$9\9" S(S$9S).5      r:\)S0S*\74S+ jj5       r;\)S0S*\74S, jj5       r<\)S0S*\74S- jj5       r=\)S0S*\74S. jj5       r>\)S0S*\74S/ jj5       r?g)2a(  Global Context ViT

From scratch implementation of GCViT in the style of timm swin_transformer_v2_cr.py

Global Context Vision Transformers -https://arxiv.org/abs/2206.09959

@article{hatamizadeh2022global,
  title={Global Context Vision Transformers},
  author={Hatamizadeh, Ali and Yin, Hongxu and Kautz, Jan and Molchanov, Pavlo},
  journal={arXiv preprint arXiv:2206.09959},
  year={2022}
}

Free of any code related to NVIDIA GCVit impl at https://github.com/NVlabs/GCVit.
The license for this code release is Apache 2.0 with no commercial restrictions.

However, weight files adapted from NVIDIA GCVit impl ARE under a non-commercial share-alike license
(https://creativecommons.org/licenses/by-nc-sa/4.0/) until I have a chance to train new ones...

Hacked together by / Copyright 2022, Ross Wightman
    N)partial)CallableListOptionalTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathcalculate_drop_path_rates	to_2tuple	to_ntupleMlpClassifierHeadLayerNorm2d
LayerScaleget_attnget_act_layerget_norm_layer
RelPosBias_assert   )build_model_with_cfg)feature_take_indices)register_notrace_function)named_apply
checkpoint)register_modelgenerate_default_cfgsGlobalContextVitc                      ^  \ rS rSrSrSSSS\R                  SS4S\S\\   S	\	S
\
S\S\\R                     4U 4S jjjrS rSrU =r$ )MbConvBlock6   zQA depthwise separable / fused mbconv style residual block with SE, `no norm.
    Ng      ?seFin_chsout_chsexpand_ratio
attn_layerbias	act_layerc	                   > XxS.n	[         TU ]  5         [        SSU0U	D6n
[        U[        5      (       a  US:X  d  US:X  a
  SU
S'   SU
S'   [        U5      nU=(       d    Un[        X1-  5      n[        R                  " XS	S
S
4XS.U	D6U l	        U" 5       U l
        U" U40 U
D6U l        [        R                  " XS
S
S4SU0U	D6U l        g )Ndevicedtyper,   r&   ecag      ?rd_ratioFr+      r   )groupsr+   r    )super__init__dict
isinstancestrr   intnnConv2dconv_dwactr&   conv_pw)selfr'   r(   r)   r*   r+   r,   r/   r0   ddattn_kwargsmid_chs	__class__s               P/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/gcvit.pyr7   MbConvBlock.__init__9   s     /5Y5"5j#&&:+=uAT&*K
#"'Kj)
#Vl+,yy!QZ&ZWYZ;W44yy1aLLL    c                     UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nX-   nU$ N)r>   r?   r&   r@   )rA   xshortcuts      rF   forwardMbConvBlock.forwardS   sG    LLOHHQKGGAJLLOLrH   )r?   r>   r@   r&   )__name__
__module____qualname____firstlineno____doc__r<   GELUr;   r   floatr:   boolr   Moduler7   rM   __static_attributes____classcell__rE   s   @rF   r$   r$   6   s    
 &*"%")+MM c]M  	M
 M M BIIM M4 rH   r$   c                      ^  \ rS rSrSS\R
                  \SS4S\S\\   S\	S\
\R                     S\
\R                     4
U 4S	 jjjrS
 rSrU =r$ )Downsample2d]   Nconvdimdim_out	reductionr,   
norm_layerc                   > XgS.n[         T	U ]  5         U=(       d    UnUb	  U" U40 UD6O[        R                  " 5       U l        [        U4SU0UD6U l        US;   d   eUS:X  a"  [        R                  " XSSS4SS	0UD6U l        OIUS
:X  a#  X:X  d   e[        R                  " SSSS9U l        O X:X  d   e[        R                  " SS9U l        Ub  U" U40 UD6U l        g [        R                  " 5       U l        g )Nr.   r,   )r^   maxavgr^   r3      r   r+   Frd   kernel_sizestridepaddingrh   )r6   r7   r<   Identitynorm1r$   
conv_blockr=   ra   	MaxPool2d	AvgPool2dnorm2)
rA   r_   r`   ra   r,   rb   r/   r0   rB   rE   s
            rF   r7   Downsample2d.__init__^   s     /.S.8.DZ*r*"++-
%cEYE"E2222YYsQ1O5OBODN%>!>\\a1MDN>!>\\a8DN2<2HZ.2.
bkkm
rH   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ rJ   )rm   rn   ra   rq   rA   rK   s     rF   rM   Downsample2d.forwardy   s=    JJqMOOANN1JJqMrH   )rn   rm   rq   ra   )rO   rP   rQ   rR   r<   rT   r   r;   r   r:   r   rW   r7   rM   rX   rY   rZ   s   @rF   r\   r\   ]   sz     &*#)+*5\\ c]\ 	\
 BII\ RYY\ \6 rH   r\   c            
       ~   ^  \ rS rSrSS\R
                  SS4S\S\S\S\\R                     4U 4S	 jjjr
S
 rSrU =r$ )FeatureBlock   r   rd   Nr_   levelsra   r,   c           	        > XVS.n[         TU ]  5         Un[        SU5      nUS:X  a  [        [        R
                  SS9n	O[        [        R                  SSSS9n	[        R                  " 5       U l        [        U5       Hf  n
U R                  R                  SU
S-    3[        U4S	U0UD65        U(       d  M:  U R                  R                  S
U
S-    3U	" 5       5        US-  nMh     g )Nr.   r   re   rf   rk   r3   rg   r^   r,   pool)r6   r7   rd   r   r<   rp   ro   
Sequentialblocksrange
add_moduler$   )rA   r_   ry   ra   r,   r/   r0   rB   
reductionspool_fnirE   s              rF   r7   FeatureBlock.__init__   s     /
Qbll:Gbll!QOGmmovAKK""T!A#<S1\I1\Y[1\]z&&acU|WY?a
	 rH   c                 $    U R                  U5      $ rJ   r}   rt   s     rF   rM   FeatureBlock.forward   s    {{1~rH   r   )rO   rP   rQ   rR   r<   rT   r;   r:   r   rW   r7   rM   rX   rY   rZ   s   @rF   rw   rw      sZ     ")+     	 
 BII   0 rH   rw   c            
          ^  \ rS rSrSS\R
                  \SS4S\S\S\\R                     S\\R                     4U 4S	 jjjr
S
 rSrU =r$ )Stem   r3   `   Nr'   r(   r,   rb   c                    > [         TU ]  5         XVS.n[        R                  " X4SSSS.UD6U l        [        U4X4S.UD6U l        g )Nr.   r3   rf   r   rg   )r,   rb   )r6   r7   r<   r=   conv1r\   down)	rA   r'   r(   r,   rb   r/   r0   rB   rE   s	           rF   r7   Stem.__init__   sO     	/YYvYAaQRYVXY
 [I[XZ[	rH   c                 J    U R                  U5      nU R                  U5      nU$ rJ   r   r   rt   s     rF   rM   Stem.forward   s!    JJqMIIaLrH   r   )rO   rP   rQ   rR   r<   rT   r   r;   r   rW   r7   rM   rX   rY   rZ   s   @rF   r   r      sk     )+*5\\ \ BII	\
 RYY\ \ rH   r   c                      ^  \ rS rSr      SS\S\S\\\4   S\S\S\S\4U 4S	 jjjrSS
\	\
R                     4S jjrSrU =r$ )WindowAttentionGlobal   r_   	num_headswindow_size
use_globalqkv_bias	attn_drop	proj_dropc
                   > XS.n
[         TU ]  5         [        U5      nX0l        X l        X-  U l        U R
                  S-  U l        X@l        [        SX2S.U
D6U l	        U R                  (       a"  [        R                  " XS-  4SU0U
D6U l        O![        R                  " XS-  4SU0U
D6U l        [        R                  " U5      U l        [        R                  " X40 U
D6U l        [        R                  " U5      U l        g )Nr.   g      )r   r   rf   r+   r3   r5   )r6   r7   r   r   r   head_dimscaler   r   rel_posr<   LinearqkvDropoutr   projr   )rA   r_   r   r   r   r   r   r   r/   r0   rB   rE   s              rF   r7   WindowAttentionGlobal.__init__   s     /,&"(]]d*
$!UkURTU??yyAgCHCCDHyyAgCHCCDHI.IIc-"-	I.rH   q_globalc                     UR                   u  p4nU R                  (       a  Ub  [        UR                   S   UR                   S   :H  S5        U R                  U5      nUR	                  X4SU R
                  U R                  5      R                  SSSSS5      nUR                  S5      u  pxUR                  X2R                   S   -  SSS5      n	U	R	                  X4U R
                  U R                  5      R                  SSSS5      n	O^U R                  U5      R	                  X4SU R
                  U R                  5      R                  SSSSS5      n
U
R                  S5      u  pnXR                  -  n	XR                  SS5      R                  5       -  nU R                  U5      nUR                  SS	9nU R                  U5      nX-  R                  SS5      R	                  X4U5      nU R!                  U5      nU R#                  U5      nU$ )
Nz*x and q_global seq lengths should be equalrf   r   r3   r      )r_   )shaper   r   r   reshaper   r   permuteunbindrepeatr   	transpose
contiguousr   softmaxr   r   r   )rA   rK   r   BNCkvkvqr   attns               rF   rM   WindowAttentionGlobal.forward   s   ''a??x3AGGBK8>>"#557cd!BA!T^^T]]CKKAqRSUVXYZB99Q<DA^^A%6 61a@A		!>FFq!QPQRA((1+%%aAt~~t}}MUUVWYZ\]_`bcdCjjmGA!

N;;r2&1133||D!|||#~~d#X  A&..qQ7IIaLNN1rH   )
r   r   r   r   r   r   r   r   r   r   )TT        r   NNrJ   )rO   rP   rQ   rR   r;   r   rV   rU   r7   r   torchTensorrM   rX   rY   rZ   s   @rF   r   r      s      $!!!// / sCx	/
 / / / / /<8ELL#9  rH   r   r   c                     U R                   u  p#pEU R                  X#US   -  US   XAS   -  US   U5      n U R                  SSSSSS5      R                  5       R                  SUS   US   U5      nU$ )Nr   r   r3   rf   r      r   r   viewr   r   )rK   r   r   HWr   windowss          rF   window_partitionr      s    JA!	q{1~%{1~qN7JKXYN\]^Aii1aAq)446;;BAP[\]P^`abGNrH   img_sizec                     Uu  p4U R                   S   nU R                  SX1S   -  XAS   -  US   US   U5      nUR                  SSSSSS5      R                  5       R                  SX4U5      nU$ )Nr   r   r   r3   rf   r   r   r   )r   r   r   r   r   r   rK   s          rF   window_reverser      s~    DAbAR!n,aq>.A;q>S^_`SacdeA			!Q1a#..055b!BAHrH   c                   F  ^  \ rS rSrSSSSSSSS\\R                  \R                  SS4S\S\	\\4   S	\S
\S\
S\S\S\\
   S\
S\
S\
S\S\\R                     S\\R                     4U 4S jjjrSS\\R$                     4S jjrSS\\R$                     4S jjrSrU =r$ )GlobalContextVitBlock            @TNr   r_   	feat_sizer   r   	mlp_ratior   r   layer_scaler   r   	drop_pathr*   r,   rb   c           
        > UUS.n[         TU ]  5         [        U5      n[        U5      nX@l        [	        US   US   -  US   US   -  -  5      U l        U" U40 UD6U l        U" U4UUUUU
U	S.UD6U l        Ub  [        X40 UD6O[        R                  " 5       U l        US:  a  [        U5      O[        R                  " 5       U l        U" U40 UD6U l        [        SU[	        X-  5      XS.UD6U l        Ub  [        X40 UD6O[        R                  " 5       U l        US:  a  [        U5      U l        g [        R                  " 5       U l        g )Nr.   r   r   )r   r   r   r   r   r   r   )in_featureshidden_featuresr,   dropr5   )r6   r7   r   r   r;   num_windowsrm   r   r   r<   rl   ls1r   
drop_path1rq   r   mlpls2
drop_path2)rA   r_   r   r   r   r   r   r   r   r   r   r   r*   r,   rb   r/   r0   rB   rE   s                     rF   r7   GlobalContextVitBlock.__init__   sU   & /i(	,&	!A >9Q<S^_`SaCabc*r*
	
#!	
 	
	 :E9P:c5"5VXVaVaVc1:R(9-R[[]*r*
x3CO8LXaxuwx9D9P:c5"5VXVaVaVc1:R(9-R[[]rH   r   c                     UR                   u  p4pV[        XR                  5      nUR                  SU R                  S   U R                  S   -  U5      nU R	                  Xr5      n[        XR                  XE45      nU$ )Nr   r   r   )r   r   r   r   r   r   )	rA   rK   r   r   r   r   r   x_winattn_wins	            rF   _window_attn"GlobalContextVitBlock._window_attn,  ss    WW
a $4$45

2t//2T5E5Ea5HH!L99U-8%5%5v>rH   c           
         XR                  U R                  U R                  U R                  U5      U5      5      5      -   nXR	                  U R                  U R                  U R                  U5      5      5      5      -   nU$ rJ   )r   r   r   rm   r   r   r   rq   )rA   rK   r   s      rF   rM   GlobalContextVitBlock.forward4  sa    ):):4::a=()S TUU$**Q-)@ ABBrH   )
r   r   r   r   r   r   rm   rq   r   r   rJ   )rO   rP   rQ   rR   r   r<   rT   	LayerNormr;   r   rU   rV   r   r   r   rW   r7   r   r   r   rM   rX   rY   rZ   s   @rF   r   r      s$     !!#!+/!!!#8)+*,,,#+S+S S#X+S 	+S
 +S +S +S +S "%+S +S +S +S !+S BII+S RYY+S +SZ(> 8ELL#9  rH   r   c            $       6  ^  \ rS rSrSSSSSSSSS\R
                  \R                  \SS4S\S\S	\S
\	\\4   S\	\\4   S\
S\
S\
S\S\
S\\   S\S\S\\\   \4   S\\R                      S\\R                      S\\R                      4"U 4S jjjrS rSrU =r$ )GlobalContextVitStagei:  TFr   Nr   r_   depthr   r   r   
downsampleglobal_norm
stage_normr   r   r   r   r   r   r,   rb   norm_layer_clc                 &  > UUS.n[         TU ]  5         U(       a-  [        SUUS-  US.UD6U l        US-  nUS   S-  US   S-  4nO[        R
                  " 5       U l        X@l        [        U5      n[        [        R                  " [        U5      [        U5      -  5      5      n[        UU40 UD6U l        U(       a	  U" U40 UD6O[        R
                  " 5       U l        [        R                  " [!        U5       Vs/ s H<  n[#        SUUUUU	U
US-  S:g  UUU[%        U[&        5      (       a  UU   OUUUS.UD6PM>     sn5      U l        U(       a	  U" U40 UD6O[        R
                  " 5       U l        Xl        X@l        SU l        g s  snf )	Nr.   rf   )r_   r`   rb   r   r   )r_   r   r   r   r   r   r   r   r   r   r   r,   rb   Fr5   )r6   r7   r\   r   r<   rl   r   r   r;   mathlog2minrw   global_blockr   
ModuleListr~   r   r9   listr}   normr_   grad_checkpointing)rA   r_   r   r   r   r   r   r   r   r   r   r   r   r   r   r,   rb   r   r/   r0   rB   feat_levelsr   rE   s                          rF   r7   GlobalContextVitStage.__init__;  s   , /* a% 	DO 'C"1*IaLA,=>I kkmDO",$))C	NS5E$EFG(k@R@7B=33mm" 5\#%
" "! " ##'#!EQJ'##*4Y*E*E)A,9#(   "#%
 & 1;M#,,	""'-%
s   AFc                    U R                  U5      nU R                  U5      nUR                  SSSS5      nU R                  UR                  SSSS5      5      nU R                   HM  nU R
                  (       a1  [        R                  R                  5       (       d  [        X1U5      nME  U" X5      nMO     U R                  U5      nUR                  SSSS5      R                  5       nU$ )Nr   rf   r3   r   )r   r   r   r   r}   r   r   jitis_scriptingr   r   r   )rA   rK   global_queryblks       rF   rM   GlobalContextVitStage.forward}  s    OOA((+ IIaAq!''(<(<Q1a(HI;;C&&uyy/E/E/G/Gs|4(	 
 IIaLIIaAq!,,.rH   )r}   r_   r   r   r   r   r   r   )rO   rP   rQ   rR   r<   rT   r   r   r;   r   rV   rU   r   r	   r   r   rW   r7   rM   rX   rY   rZ   s   @rF   r   r   :  sA     $ %$!!+/!!36)+*,,,-8)@(@( @( 	@(
 S#X@( sCx@( @( @( @( @( @( "%@( @( @( T%[%/0@(  BII!@(" RYY#@($  		?%@( @(D rH   r   c            +         ^  \ rS rSr                       S/S\S\S\S\\\\\4   4   S\\S4   S\\\\\S4   4      S	\S
\\S4   S\\S4   S\	S\
S\\	   S\	S\	S\	S\	S\S\S\S\S\	4*U 4S jjjrS0S jr\R                  R                  S 5       r\R                  R                  S1S j5       r\R                  R                  S2S j5       r\R                  R                  S\R(                  4S j5       rS3S\S\\   4S jjr     S4S \R.                  S!\\\\\   4      S"\
S#\
S$\S%\
S\\\R.                     \\R.                  \\R.                     4   4   4S& jjr   S5S!\\\\   4   S'\
S(\
4S) jjrS \R.                  S\R.                  4S* jrS1S+\
4S, jjrS \R.                  S\R.                  4S- jrS.rU =r$ )6r"   i  in_chansnum_classesglobal_poolr   window_ratio.r   	embed_dimdepthsr   r   r   r   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rateweight_initr,   rb   r   norm_epsc                   > [         T!U ]  5         UUS.n[        U5      n[        [	        U5      US9n[        [	        U5      US9n/ U l        [        U5      n[        S U 5       5      nX0l        X l	        Xl
        Xl        [        U5      n[        USUS-
  -  -  5      =U l        U l        Ub  [!        U5      " U5      nO>Uc   e[        [!        U5      " U5       Vs/ s H  nUS   U-  US   U-  4PM     sn5      n[#        SUUUUS.UD6U l        ['        UUSS	9n/ n[)        U5       H  nUUS-
  :H  nS[+        US-
  S5      -  n UR-                  [/        S0 S
UU -  _SUU   _SU	U   _SUS   U -  US   U -  4_SUU   _SUS:g  _SU_SU
_SU_SU_SU_SU_SUU   _SU_SU_SU_UD65        U =R
                  [1        US   R2                  SUS-   -  SU 3S9/-  sl        M     [4        R6                  " U6 U l        [;        U R                  U4X=S.UD6U l        U(       a  [?        [        U R@                  US9U 5        g g s  snf ) Nr.   )epsc              3   *   #    U  H	  oS -  v   M     g7f)r   Nr5   ).0ds     rF   	<genexpr>,GlobalContextVit.__init__.<locals>.<genexpr>  s     3(Qq&(s   rf   r   r   )r'   r(   r,   rb   T)	stagewiser_   r   r   r   r   r   r   r   r   r   r   r   r   r,   rb   r   r   zstages.)num_chsra   module	pool_typer  )schemer5   )!r6   r7   r   r   r   feature_infor   tupler   r   r   r  lenr;   num_featureshead_hidden_sizer   r   stemr   r~   rd   appendr   r8   r_   r<   r|   stagesr   headr   _init_weights)"rA   r   r   r   r   r   r   r  r  r   r   r   r   r  r  r  r  r  r,   rb   r   r  r/   r0   rB   r   
num_stagesrdprr  r   
last_stagestage_scalerE   s"                                    rF   r7   GlobalContextVit.__init__  s   4 	/!),	^J7XF
} =8LX&3(33	&& "[
47	A*WX.DY8Y4ZZD1"#J/<K+++yYcOdeqOr sOr!(1+"2HQK14D!EOr stK 
!	

 
	 ($Oz"Aj1n,Js1q5!},KMM/ +Qi $A, %Q<;6	!8ST	
 (N 6 & $ " ( ) ) a& $ &  ,# & $vbz~~QqS\cdecfZg"h!ii- #. mmV, #4#4#4kt[tqst	 2 2;GN Q !ts   I
c                 n   US:X  a  [        U[        R                  5      (       a  [        R                  R	                  UR
                  5        UR                  bY  SU;   a)  [        R                  R                  UR                  SS9  g [        R                  R                  UR                  5        g g g [        U[        R                  5      (       a`  [        R                  R                  UR
                  SS9  UR                  b*  [        R                  R                  UR                  5        g g g )Nvitr   gư>)stdg{Gz?)	r9   r<   r   initxavier_uniform_weightr+   normal_zeros_)rA   r  namer  s       rF   r  GlobalContextVit._init_weights  s    U?&")),,''6;;*}>v{{3	 + - &")),,37;;*GGNN6;;/ + -rH   c                    ^ U R                  5        V^Vs1 s H$  u  mn[        U4S jS 5       5      (       d  M"  TiM&     snn$ s  snnf )Nc              3   ,   >#    U  H	  oT;   v   M     g 7frJ   r5   )r  nr   s     rF   r  3GlobalContextVit.no_weight_decay.<locals>.<genexpr>  s     S#Ra6#Rs   )relative_position_bias_tablezrel_pos.mlp)named_parametersany)rA   r   _s    ` rF   no_weight_decay GlobalContextVit.no_weight_decay  sJ     //1U1$!QS#RSS 1U 	U Us
   !AAc                     [        SSS9nU$ )Nz^stemz^stages\.(\d+))r  r}   )r8   )rA   coarsematchers      rF   group_matcherGlobalContextVit.group_matcher  s    $
 rH   c                 6    U R                    H	  nXl        M     g rJ   )r  r   )rA   enabless      rF   set_grad_checkpointing'GlobalContextVit.set_grad_checkpointing  s    A#)  rH   returnc                 .    U R                   R                  $ rJ   )r  fc)rA   s    rF   get_classifierGlobalContextVit.get_classifier
  s    yy||rH   c                     X4S.nXl         Uc   U R                  R                  R                  n[	        U R
                  U4X R                  S.UD6U l        g )Nr.   r  )r   r  r   r  r   r  r  )rA   r   r   r/   r0   rB   s         rF   reset_classifier!GlobalContextVit.reset_classifier  sP    /&))//99K"4#4#4ky[drdryvxy	rH   rK   indicesr   
stop_early
output_fmtintermediates_onlyc                    US;   d   S5       e/ n[        [        U R                  5      U5      u  pU R                  U5      n[        R
                  R                  5       (       d  U(       d  U R                  n
OU R                  SU	S-    n
[        U
5       H%  u  pU" U5      nX;   d  M  UR                  U5        M'     U(       a  U$ X4$ )a  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    norm: Apply norm layer to compatible intermediates
    stop_early: Stop iterating over blocks when last desired intermediate hit
    output_fmt: Shape of intermediate feature outputs
    intermediates_only: Only return intermediate features
Returns:

)NCHWzOutput shape must be NCHW.Nr   )	r   r  r  r  r   r   r   	enumerater  )rA   rK   rL  r   rM  rN  rO  intermediatestake_indices	max_indexr  feat_idxstages                rF   forward_intermediates&GlobalContextVit.forward_intermediates  s    * Y&D(DD&"6s4;;7G"Q IIaL99!!##:[[F[[)a-0F(0OHaA'$$Q'  1
   rH   
prune_norm
prune_headc                     [        [        U R                  5      U5      u  pEU R                  SUS-    U l        U(       a  U R                  SS5        U$ )z?Prune layers not required for specified intermediates.
        Nr   r    )r   r  r  rJ  )rA   rL  rZ  r[  rT  rU  s         rF   prune_intermediate_layers*GlobalContextVit.prune_intermediate_layers?  sK     #7s4;;7G"Qkk.9q=1!!!R(rH   c                 J    U R                  U5      nU R                  U5      nU$ rJ   )r  r  rt   s     rF   forward_features!GlobalContextVit.forward_featuresM  s!    IIaLKKNrH   
pre_logitsc                 P    U(       a  U R                  XS9$ U R                  U5      $ )N)rc  )r  )rA   rK   rc  s      rF   forward_headGlobalContextVit.forward_headR  s#    6@tyyy2RdiiPQlRrH   c                 J    U R                  U5      nU R                  U5      nU$ rJ   )ra  re  rt   s     rF   rM   GlobalContextVit.forwardU  s'    !!!$a rH   )
r  r  r   r  r  r   r   r  r  r  )r3     re      )    rk     rk  N@   r3   r      r   rf   r      rl  g      @TNr   r   r   r   r]  gelulayernorm2d	layernormh㈵>NN)r'  F)T)NNN)NFFrQ  F)r   FT) rO   rP   rQ   rR   r;   r:   r	   r   r   rU   rV   r7   r  r   r   ignorer8  r=  rB  r<   rW   rG  rJ  r   r   rX  r^  ra  re  rM   rX   rY   rZ   s   @rF   r"   r"     sY    #$47,<AE&3)6"!+/!$&$&$&!#+!,"1UOUO UO 	UO
 CsCx01UO  S/UO "%U38_(<"=>UO UO #s(OUO S#XUO UO UO "%UO UO "UO  "!UO" "#UO$ %UO& 'UO( )UO* +UO, -UO UOn0  YYU U
 YY  YY* * YY		  zC zhsm z 8<$$',( ||(  eCcN34(  	( 
 (  (  !%(  
tELL!5tELL7I)I#JJ	K( X ./$#	3S	>*  	%,, 5<< 
S$ S %,,  rH   c           	      <    [        [        X4S[        SSS90UD6nU$ )Nfeature_cfg)r   r   rf   r3   T)out_indicesflatten_sequential)r   r"   r8   )variant
pretrainedkwargsmodels       rF   _create_gcvitr  [  s1     '\dK E
 LrH   c                 4    U SSSSS[         [        SSSS	S
.UE$ )Nri  )r3   rj  rj  )r   r   g      ?bicubicz
stem.conv1zhead.fcTz
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationmeanr(  
first_conv
classifierfixed_input_sizelicenser
   )r  r~  s     rF   _cfgr  d  s6    4}SYI%.B")   rH   z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xxtiny_224_nvidia-d1d86009.pth)r  z|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xtiny_224_nvidia-274b92b7.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_tiny_224_nvidia-ac783954.pthz|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_small_224_nvidia-4e98afa2.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_base_224_nvidia-f009139b.pth)zgcvit_xxtiny.in1kzgcvit_xtiny.in1kzgcvit_tiny.in1kzgcvit_small.in1kzgcvit_base.in1krD  c                 8    [        SSSS.UD6n[        SSU 0UD6$ )N)rf   rf      rf   rp  r  r   r}  r5   )gcvit_xxtinyr8   r  r}  r~  model_kwargss      rF   r  r  ~  s6      L OJO,OOrH   c                 8    [        SSSS.UD6n[        SSU 0UD6$ )N)r3   r   r  r   rp  r  r}  r5   )gcvit_xtinyr  r  s      rF   r  r    s6      L N:NNNrH   c                 8    [        SSSS.UD6n[        SSU 0UD6$ )Nrn  rp  r  r}  r5   )
gcvit_tinyr  r  s      rF   r  r    s6      L M*MMMrH   c           	      >    [        SSSSSSS.UD6n[        S	SU 0UD6$ )
Nrn  )r3   r        r   rf   ru  r  r   r  r   r   r}  r5   )gcvit_smallr  r  s      rF   r  r    s?       L N:NNNrH   c           	      >    [        SSSSSSS.UD6n[        S	SU 0UD6$ )
Nrn  )r   rq  rl  rk     rf   ru  r  r}  r5   )
gcvit_baser  r  s      rF   r  r    s?       L M*MMMrH   rv  )r]  )@rS   r   	functoolsr   typingr   r   r   r   r   r	   r   torch.nnr<   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   r   	_registryr    r!   __all__rW   r$   r\   rw   r   r   r;   r   r   r   r   r"   r  r  default_cfgsr  r  r  r  r  r5   rH   rF   <module>r     s  *   ? ?   A    + + 3 0 <
$")) $N!299 !H299 :299 *8BII 8vU38_  sCx E#s(O  9BII 9xRBII RjIryy IX	 % LM KL JK KL JK&  P0@ P P O/? O O N.> N N O/? O O N.> N NrH   