
    RЦioJ                     H   S r SSKJr  SSKJrJr  SSKrSSKJr  SSKJ	r	J
r
  SSKJr  SSKJrJrJr  S	S
KJr  S	SKJrJrJr  S/r " S S\5      rS\S jrS]S jr\" 0 S\" SSS9_S\" SSS9_S\" SSS9_S\" SSSSS9_S\" SSSS 9_S!\" SS"SS 9_S#\" SS$SS 9_S%\" SS&SSSS'9_S(\" SS)S9_S*\" SS+SSS9_S,\" SS-S9_S.\" SS/S9_S0\" SS1SSS9_S2\" SS3S9_S4\" SS5SSS9_S6\" SS7S9_S8\" SS9SS:9_\" SS;SSS9\" SS<SS:9\" SS=SS:9\" SS>SSS9\" SS?SS:9\" SS@SSS9\" SSASS:9SB.E5      r\S^SC\4SD jj5       r\S^SC\4SE jj5       r\S^SC\4SF jj5       r\S^SC\4SG jj5       r\S^SC\4SH jj5       r \S^SC\4SI jj5       r!\S^SC\4SJ jj5       r"\S^SC\4SK jj5       r#\S^SC\4SL jj5       r$\S^SC\4SM jj5       r%\S^SC\4SN jj5       r&\S^SC\4SO jj5       r'\S^SC\4SP jj5       r(\S^SC\4SQ jj5       r)\S^SC\4SR jj5       r*\S^SC\4SS jj5       r+\" \,S8STSUSVSWSXSYSZS[.5        g)_aZ  DeiT - Data-efficient Image Transformers

DeiT model defs and weights from https://github.com/facebookresearch/deit, original copyright below

paper: `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877

paper: `DeiT III: Revenge of the ViT` - https://arxiv.org/abs/2204.07118

Modifications copyright 2021, Ross Wightman
    )partial)OptionalTypeN)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)resample_abs_pos_embed)VisionTransformertrunc_normal_checkpoint_filter_fn   )build_model_with_cfg)generate_default_cfgsregister_modelregister_model_deprecationsVisionTransformerDistilledc                   T  ^  \ rS rSrSrU 4S jrSU 4S jjr\R                  R                  SS j5       r
\R                  R                  S\R                  4S j5       rSS\S	\\   4S
 jjr\R                  R                  SS j5       rS rSS\S\R*                  4S jjrSrU =r$ )r      zVision Transformer w/ Distillation Token and Head

Distillation token & head support for `DeiT: Data-efficient Image Transformers`
    - https://arxiv.org/abs/2012.12877
c                   > UR                  SS5      n[        TU ]  " U0 UDSS0D6  U R                  S;   d   eUR	                  SS 5      UR	                  SS 5      S.nSU l        [        R                  " [        R                  " S	S	U R                  40 UD65      U l        [        R                  " [        R                  " S	U R                  R                  U R
                  -   U R                  40 UD65      U l        U R                  S
:  a,  [        R                   " U R                  U R                  40 UD6O[        R"                  " 5       U l        SU l        US:X  a  SOUU l        US:w  a  U R+                  SS9  g g )Nweight_init skip)tokendevicedtype)r   r      r   r   Freset)needs_reset)popsuper__init__global_poolgetnum_prefix_tokensr   	Parametertorchempty	embed_dim
dist_tokenpatch_embednum_patches	pos_embednum_classesLinearIdentity	head_distdistilled_trainingweight_init_modeinit_weights)selfargskwargsr   dd	__class__s        O/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/deit.pyr"   #VisionTransformerDistilled.__init__#   sD   jj3$=&=f=:---

8T2VZZQU=VW!",,u{{1a'N2'NOKK4++77$:P:PPRVR`R`gdfgiNRN^N^abNb4>>43C3CJrJhjhshshu"'+6&+@k& %0 !    c                 r   > U=(       d    U R                   n[        U R                  SS9  [        TU ]  XS9  g )Ng{Gz?)std)moder   )r3   r   r*   r!   r4   )r5   r?   r   r9   s      r:   r4   'VisionTransformerDistilled.init_weights5   s1    ,t,,doo3/$@r<   c                     [        SSS/S9$ )Nz+^cls_token|pos_embed|patch_embed|dist_token)z^blocks\.(\d+)N)z^norm)i )stemblocks)dict)r5   coarses     r:   group_matcher(VisionTransformerDistilled.group_matcher:   s    ?)$&
 	
r<   returnc                 2    U R                   U R                  4$ Nheadr1   )r5   s    r:   get_classifier)VisionTransformerDistilled.get_classifierC   s    yy$..((r<   r.   r#   c                 4   Xl         US:  a!  [        R                  " U R                  U5      O[        R                  " 5       U l        US:  a1  [        R                  " U R                  U R                   5      U l        g [        R                  " 5       U l        g )Nr   )r.   r   r/   r)   r0   rL   r1   )r5   r.   r#   s      r:   reset_classifier+VisionTransformerDistilled.reset_classifierG   sc    &>IAoBIIdnnk:SUS^S^S`	HSVW4>>43C3CD]_]h]h]jr<   c                     Xl         g rJ   )r2   )r5   enables     r:   set_distilled_training1VisionTransformerDistilled.set_distilled_trainingL   s    "(r<   c                    U R                   (       am  UR                  u  p#pEU R                  R                  n[	        U R
                  X44UU R                  (       a  SOU R                  S9nUR                  USU5      nOU R
                  nU R                  (       am  X-   n[        R                  " U R                  R                  UR                  S   SS5      U R                  R                  UR                  S   SS5      U4SS9nOl[        R                  " U R                  R                  UR                  S   SS5      U R                  R                  UR                  S   SS5      U4SS9nX-   nU R                  U5      $ )Nr   )new_sizeold_sizer%   r   )dim)dynamic_img_sizeshaper+   	grid_sizer
   r-   no_embed_classr%   viewr'   cat	cls_tokenexpandr*   pos_drop)r5   xBHWCprev_grid_sizer-   s           r:   
_pos_embed%VisionTransformerDistilled._pos_embedP   sL     JA!!--77N.''+':':!@V@V	I q"a AI A		%%aggaj"b9&&qwwqz2r: 	A 		%%aggaj"b9&&qwwqz2r: 	A
 A}}Qr<   
pre_logitsc                 "   US S 2S4   US S 2S4   p1U(       a  X-   S-  $ U R                  U5      nU R                  U5      nU R                  (       a7  U R                  (       a&  [        R
                  R                  5       (       d  X4$ X-   S-  $ )Nr   r   r   )rL   r1   r2   trainingr'   jitis_scripting)r5   rd   rl   x_dists       r:   forward_head'VisionTransformerDistilled.forward_headq   s|    adGQq!tW6J!##IIaL'""t}}UYY=S=S=U=U9 J!##r<   )r*   r2   rL   r1   r.   r%   r-   r3   )r   TFrJ   )T)__name__
__module____qualname____firstlineno____doc__r"   r4   r'   ro   ignorerF   r   ModulerM   intr   strrP   rT   rj   boolTensorrr   __static_attributes____classcell__)r9   s   @r:   r   r      s    1$A
 YY
 
 YY)		 ) )kC khsm k
 YY) ) B$$ $5<< $ $r<   c           	          UR                  SS5      nU(       a  [        O[        n[        UU U4[	        [
        SS9[        USS9S.UD6nU$ )Nout_indices   T)adapt_layer_scalegetter)r   feature_cls)pretrained_filter_fnfeature_cfg)r    r   r   r   r   r   rD   )variant
pretrained	distilledr7   r   	model_clsmodels          r:   _create_deitr      s]    **]A.K.7*=NI  %%9TR[hG E Lr<   c                 4    U SSS SSS[         [        SSSS	.UE$ )
Ni  )r      r   g?bicubicTzpatch_embed.projrL   z
apache-2.0)urlr.   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizemeanr>   
first_conv
classifierlicenser   )r   r7   s     r:   _cfgr      s5    =t%.B(  r<   zdeit_tiny_patch16_224.fb_in1kztimm/zFhttps://dl.fbaipublicfiles.com/deit/deit_tiny_patch16_224-a1311bcf.pth)	hf_hub_idr   zdeit_small_patch16_224.fb_in1kzGhttps://dl.fbaipublicfiles.com/deit/deit_small_patch16_224-cd65a155.pthzdeit_base_patch16_224.fb_in1kzFhttps://dl.fbaipublicfiles.com/deit/deit_base_patch16_224-b5f2ef4d.pthzdeit_base_patch16_384.fb_in1kzFhttps://dl.fbaipublicfiles.com/deit/deit_base_patch16_384-8de9b5d1.pth)r     r   g      ?)r   r   r   r   z'deit_tiny_distilled_patch16_224.fb_in1kzPhttps://dl.fbaipublicfiles.com/deit/deit_tiny_distilled_patch16_224-b40b3cf7.pthrK   )r   r   r   z(deit_small_distilled_patch16_224.fb_in1kzQhttps://dl.fbaipublicfiles.com/deit/deit_small_distilled_patch16_224-649709d9.pthz'deit_base_distilled_patch16_224.fb_in1kzPhttps://dl.fbaipublicfiles.com/deit/deit_base_distilled_patch16_224-df68dfff.pthz'deit_base_distilled_patch16_384.fb_in1kzPhttps://dl.fbaipublicfiles.com/deit/deit_base_distilled_patch16_384-d0272ac0.pth)r   r   r   r   r   zdeit3_small_patch16_224.fb_in1kz;https://dl.fbaipublicfiles.com/deit/deit_3_small_224_1k.pthzdeit3_small_patch16_384.fb_in1kz;https://dl.fbaipublicfiles.com/deit/deit_3_small_384_1k.pthz deit3_medium_patch16_224.fb_in1kz<https://dl.fbaipublicfiles.com/deit/deit_3_medium_224_1k.pthzdeit3_base_patch16_224.fb_in1kz:https://dl.fbaipublicfiles.com/deit/deit_3_base_224_1k.pthzdeit3_base_patch16_384.fb_in1kz:https://dl.fbaipublicfiles.com/deit/deit_3_base_384_1k.pthzdeit3_large_patch16_224.fb_in1kz;https://dl.fbaipublicfiles.com/deit/deit_3_large_224_1k.pthzdeit3_large_patch16_384.fb_in1kz;https://dl.fbaipublicfiles.com/deit/deit_3_large_384_1k.pthzdeit3_huge_patch14_224.fb_in1kz:https://dl.fbaipublicfiles.com/deit/deit_3_huge_224_1k.pthz(deit3_small_patch16_224.fb_in22k_ft_in1kz<https://dl.fbaipublicfiles.com/deit/deit_3_small_224_21k.pth)r   r   r   z<https://dl.fbaipublicfiles.com/deit/deit_3_small_384_21k.pthz=https://dl.fbaipublicfiles.com/deit/deit_3_medium_224_21k.pthz;https://dl.fbaipublicfiles.com/deit/deit_3_base_224_21k.pthz;https://dl.fbaipublicfiles.com/deit/deit_3_base_384_21k.pthz<https://dl.fbaipublicfiles.com/deit/deit_3_large_224_21k.pthz<https://dl.fbaipublicfiles.com/deit/deit_3_large_384_21k.pthz>https://dl.fbaipublicfiles.com/deit/deit_3_huge_224_21k_v1.pth)(deit3_small_patch16_384.fb_in22k_ft_in1k)deit3_medium_patch16_224.fb_in22k_ft_in1k'deit3_base_patch16_224.fb_in22k_ft_in1k'deit3_base_patch16_384.fb_in22k_ft_in1k(deit3_large_patch16_224.fb_in22k_ft_in1k(deit3_large_patch16_384.fb_in22k_ft_in1k'deit3_huge_patch14_224.fb_in22k_ft_in1krH   c           	      L    [        SSSSS9n[        SSU 0[        U40 UD6D6nU$ )zDeiT-tiny model @ 224x224 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
         r   
patch_sizer)   depth	num_headsr   )deit_tiny_patch16_224rD   r   r   r7   
model_argsr   s       r:   r   r      s7    
 s"JJfZf4PZKe^dKefELr<   c           	      L    [        SSSSS9n[        SSU 0[        U40 UD6D6nU$ )zDeiT-small model @ 224x224 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r      r   r   )deit_small_patch16_224r   r   s       r:   r   r     s7    
 s"JJgjgDQ[Lf_eLfgELr<   c           	      L    [        SSSSS9n[        SSU 0[        U40 UD6D6nU$ )zDeiT base model @ 224x224 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r      r   r   r   )deit_base_patch16_224r   r   s       r:   r   r     7    
 s"KJfZf4PZKe^dKefELr<   c           	      L    [        SSSSS9n[        SSU 0[        U40 UD6D6nU$ )zDeiT base model @ 384x384 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   r   r   )deit_base_patch16_384r   r   s       r:   r   r     r   r<   c           	      P    [        SSSSS9n[         SU SS.[        U40 UD6D6nU$ )	zDeiT-tiny distilled model @ 224x224 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   r   r   Tr   r   )deit_tiny_distilled_patch16_224r   r   s       r:   r   r   "  sG    
 s"JJ)p6@DpTXYcTngmTnpELr<   c           	      P    [        SSSSS9n[         SU SS.[        U40 UD6D6nU$ )	zDeiT-small distilled model @ 224x224 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   r   r   Tr   ) deit_small_distilled_patch16_224r   r   s       r:   r   r   -  sG    
 s"JJ*q7ATqUYZdUohnUoqELr<   c           	      P    [        SSSSS9n[         SU SS.[        U40 UD6D6nU$ )zDeiT-base distilled model @ 224x224 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   r   Tr   )deit_base_distilled_patch16_224r   r   s       r:   r   r   8  G    
 s"KJ)p6@DpTXYcTngmTnpELr<   c           	      P    [        SSSSS9n[         SU SS.[        U40 UD6D6nU$ )zDeiT-base distilled model @ 384x384 from paper (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   r   Tr   )deit_base_distilled_patch16_384r   r   s       r:   r   r   C  r   r<   c           	      P    [        SSSSSSS9n[        S	SU 0[        U40 UD6D6nU$ )
zDeiT-3 small model @ 224x224 from paper (https://arxiv.org/abs/2204.07118).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   r   Tư>r   r)   r   r   r^   init_valuesr   )deit3_small_patch16_224r   r   s       r:   r   r   N  =    
 s"Z^lpqJhzhTR\Mg`fMghELr<   c           	      P    [        SSSSSSS9n[        S	SU 0[        U40 UD6D6nU$ )
zDeiT-3 small model @ 384x384 from paper (https://arxiv.org/abs/2204.07118).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   r   Tr   r   r   )deit3_small_patch16_384r   r   s       r:   r   r   X  r   r<   c           	      P    [        SSSSSSS9n[        S	SU 0[        U40 UD6D6nU$ )
zDeiT-3 medium model @ 224x224 (https://arxiv.org/abs/2012.12877).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   i   r      Tr   r   r   )deit3_medium_patch16_224r   r   s       r:   r   r   b  s=    
 s"Z^lpqJi
idS]NhagNhiELr<   c           	      P    [        SSSSSSS9n[        SSU 0[        U40 UD6D6nU$ )	zDeiT-3 base model @ 224x224 from paper (https://arxiv.org/abs/2204.07118).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   Tr   r   r   )deit3_base_patch16_224r   r   s       r:   r   r   l  =    
 s"[_mqrJgjgDQ[Lf_eLfgELr<   c           	      P    [        SSSSSSS9n[        SSU 0[        U40 UD6D6nU$ )	DeiT-3 base model @ 384x384 from paper (https://arxiv.org/abs/2204.07118).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   Tr   r   r   )deit3_base_patch16_384r   r   s       r:   r   r   v  r   r<   c           	      P    [        SSSSSSS9n[        SSU 0[        U40 UD6D6nU$ )	zDeiT-3 large model @ 224x224 from paper (https://arxiv.org/abs/2204.07118).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r         Tr   r   r   )deit3_large_patch16_224r   r   s       r:   r   r     =    
 t2\`nrsJhzhTR\Mg`fMghELr<   c           	      P    [        SSSSSSS9n[        SSU 0[        U40 UD6D6nU$ )	zDeiT-3 large model @ 384x384 from paper (https://arxiv.org/abs/2204.07118).
ImageNet-1k weights from https://github.com/facebookresearch/deit.
r   r   r   Tr   r   r   )deit3_large_patch16_384r   r   s       r:   r   r     r   r<   c           	      P    [        SSSSSSS9n[        S	SU 0[        U40 UD6D6nU$ )
r      i       r   Tr   r   r   )deit3_huge_patch14_224r   r   s       r:   r   r     s=    
 t2\`nrsJgjgDQ[Lf_eLfgELr<   r   r   r   r   r   r   r   ) deit3_small_patch16_224_in21ft1k deit3_small_patch16_384_in21ft1k!deit3_medium_patch16_224_in21ft1kdeit3_base_patch16_224_in21ft1kdeit3_base_patch16_384_in21ft1k deit3_large_patch16_224_in21ft1k deit3_large_patch16_384_in21ft1kdeit3_huge_patch14_224_in21ft1k)FF)r   rt   )-ry   	functoolsr   typingr   r   r'   r   	timm.datar   r	   timm.layersr
   timm.models.vision_transformerr   r   r   _builderr   	_registryr   r   r   __all__r   r   r   default_cfgsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ru    r<   r:   <module>r      s  	  !   A . a a * Y Y'
(`$!2 `$F	 % ^&#TT&V^&
 %dU'W^& $TT&V^& $TT 3&0^&  .t^(0*!^&( /_(1*)^&0 .t^(0*1^&8 .t^ 3(	0*9^&D &tI(KE^&J &tI 3(0K^&R 'J)LS^&X %dH'JY^&^ %dH 3'0_^&f &tI(Kg^&l &tI 3(0m^&t %dH'Ju^&| /J1}^&D 15J 310 26K2 04I0 04I 300 15J1 15J 310 04L0u^& ^B 9J   :K   9J   9J   C]   D^   C]   C]   ;L   ;L   <M   :K   :K   ;L   ;L   :K   H(R(R)T'P'P(R(R'P	' 	r<   