RЦix&z<SrSSKrSSKrSSKrSSKrSSKJr SSKJr SSK J r J r J r J r JrJrJrJrJr SSK Jr SSKrSSKJr SSKJs Jr SSKJr SSKJrJ r J!r!J"r"J#r#J$r$ SS K%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3J4r4J5r5J6r6J7r7J8r8J9r9J:r:J;r; S S KJ?r? S S K@JArAJBrBJCrCJDrD S SKEJFrFJGrGJHrH S/rI\R"\K5rL\&\&\'S.rMGSS\:S\NS\NS\OS\OS\OS\OS\PS\PS\ \\RS\NS\R4SjjrR"SS \R5rS"S!S"\R5rT"S#S$\R5rU"S%S&\R5rV"S'S(\R5rWGSS)\RS*\YS+\NS,\O4S-jjrZ"S.S\R5r[GSS0\RS1\YS2\OSS4S3jjr\GSS0\RS1\YS4\PS2\OSS4 S5jjr]GSS0\RS1\YS2\OSS4S6jjr^GSS0\RS1\YS2\OSS4S7jjr_GSS8\YS4\PS2\OS\ 4S9jjr`GSS;\RS<\RS+\NS=\\N\N4S>\YS?\OS\R4S@jjra\R"5GSSA\[SB\YSC\YSD\OSS4 SEjj5rcGSSF\ \Y\R4SA\[SC\YS\ \Y\R44SGjjrdSF\ \Y\R4SA\[S\ \Y\R44SHjreSF\ \Y\R4SA\[S\ \Y\R44SIjrfSF\g4SJjrhGSSF\ \Y\R4SA\[SK\OS>\YS?\OS\ \Y\R44 SLjjriGSSM\YS\ \Y\ 44SNjjrj0SO\j"SPSQ9_SR\j"SPSQ9_SS\j"STSPSSU9_SV\j"SWSPSSXSYSZ9_S[\j"S\SPSSU9_S]\j"S^SPSSXSYSZ9_S_\j"S`SPSSU9_Sa\j"SbSPSSXSYSZ9_Sc\j"SdSPSSU9_Se\j"SfSPSSXSYSZ9_Sg\j"ShSPSSU9_Si\j"SjSPSSXSYSZ9_Sk\j"SlSPSSU9_Sm\j"SnSPSSU9_So\j"SpSPSSXSYSZ9_Sq\j"SrSPSs9_St\j"SuSPSXSYSv9_0Sw\j"SxSPSXSYSv9_Sy\j"SzSPSSU9_S{\j"S|SPSSXSYSZ9_S}\j"S~SPSSU9_S\j"SSPSSXSYSZ9_S\j"SSPSSU9_S\j"SSPSSXSYSZ9_S\j"S/S9_S\j"S/S9_S\j"S/S9_S\j"S/S9_S\j"SPSS9_S\j"SPSS9_S\j"SPSS9_S\j"SPSS9_S\j"SPSS9_S\j"SSPSSS9_E0S\j"SSPSSS9_S\j"SSPSSS9_S\j"SSPSSS9_S\j"SSPSSS9_S\j"SSPSSS9_S\j"SSPSSS9_S\j"SSSPS9_S\j"SSSPS9_S\j"SSP\\ SS9_S\j"SSP\\ SS9_S\j"SSP\\ SS9_S\j"SSP\\ SS9_S\j"SSPS\\ SSSYS9_S\j"SSPS\\ SSSYS9_S\j"SSPS\\ SSSYS9_S\j"SSPS\\ SSSYS9_S\j"SSPS\\ SSSYS9_E0S\j"SSPS\\ SSSYS9_S\j"SSPS\\ SSSYS9_S\j"SSPS\\ SSSYS9_S\j"SSPSSSSSS9_S\j"SSPSSSSS9_S\j"SSPSs9_S\j"SPSSSS9_S\j"SPSSS9_S\j"SPSXSSS9_S\j"SSS9_S\j"5_S\j"SP\#\$S9_S\j"SP\#\$SYSXS9_S\j"SP\#\$SYSS9_S\j"SP\#\$SS9_S\j"SP\#\$SYSXSS9_S\j"SP\!\"SYS9_E0S\j"SP\!\"SYSSS9_S\j"SP\#\$SYS9_S\j"SP\#\$SYSSS9_S\j"\#\$S9_S\j"SP\#\$SSXSS9_S\j"SP\#\$SS9_S\j"SP\#\$SSXSS9_S\j"SP\#\$SYS9_S\j"SP\#\$SYSSS9_S\j"SP\#\$S9_S\j"SP\#\$SYS9_S\j"SP\#\$SYSXSS9_S\j"SP\!\"SYS9_S\j"SP\!\"SYSSS9_S\j"SP\#\$SYS9_S\j"S/\#\$SYSSS9_S\j"SP\#\$S9_E0S\j"SP\#\$S9_S\j"SP\#\$SYSXSS9_S\j"SP\#\$SYS9_S\j"SP\#\$SS9_S\j"SP\!\"SYSGS9_GS\j"SP\#\$SYSGS9_GS\j"SP\#\$SS9_GS\j"SP\#\$SYSGS9_GS\j"SP\#\$GSS9_GS\j"SP\#\$SYGSGS9_GS\j"SP\!\"SYGSGS9_GS \j"SP\#\$SYGS GS9_GS \j"SP\#\$SYGS GS9_GS \j"SP\#\$SYGS GS9_GS\j"SPGSGS\#\$GSGS9_GS\j"SPGS\#\$SYGSGS9_GS\j"SPGS\#\$SSYGSGS9_E0GS\j"SPGS\#\$SYGSGS9_GS\j"SP\#\$SYGSGS9_GS\j"SP\#\$SYSGSGS9_GS\j"SP\#\$SYGSGS9_GS\j"SP\#\$SYGSGS9_GS\j"SPGS\#\$SYGSGS9_GS\j"SPGS\#\$SYGSGS9_GS \j"SPGSGS\#\$SYGSGS!9_GS"\j"SPGSGS\#\$SYGS GS!9_GS#\j"SP\#\$GSGSSYGS$GS GS%9_GS&\j"SPGS'GS\#\$SYGS GS!9_GS(\j"SPGS'\#\$GS$SYSGS GS)9_GS*\j"SPGS'\#\$SYGS GS9_GS+\j"SPGS'\#\$GS$SYSGS GS)9_GS,\j"SPGS'GS\#\$SYGSGS!9_GS-\j"SPGS'GS\#\$SYGSGS!9_GS.\j"SPGS'GS\#\$SYGSGS!9_E0GS/\j"SPGS'GS\#\$SYGS GS!9_GS0\j"SPGS'\#\$SYGS GS9_GS1\j"SPGS'GS\#\$SYGS GS!9_GS2\j"SPGS'GS\#\$SYGSGS!9_GS3\j"SPGS'GS\#\$SYGSGS!9_GS4\j"SPGS'GS\#\$SYGSGS!9_GS5\j"SPGS\#\$GSGS69_GS7\j"SPGS\#\$GSGS69_GS8\j"SPGS\#\$SYGSGS99_GS:\j"SPGS\#\$SYSGSGS;9_GS<\j"SPGS\#\$SYGS=GS>9_GS?\j"S/SSGS@9_GSA\j"S/SSGS@9_GSB\j"S/S9_GSC\j"S/S9_GSD\j"S/S9_GSE\j"SPGS\#\$GSFSYGSG9_E0GSH\j"SPGS\#\$SSYSGSI9_GSJ\j"SPGS\#\$GSFSYGSG9_GSK\j"SPGS\#\$SSYSGSI9_GSL\j"GSMSSPSSGSN9_GSO\j"GSPSSPSSGSN9_GSQ\j"GSRSSPSSGSN9_GSS\j"GSTSSPSSGSN9_GSU\j"GSVSSPSSGSN9_GSW\j"GSXSSPSSGSN9_GSY\j"GSZSSPSSSGS[9_GS\\j"GS]SSPSSSGS[9_GS^\j"GS_SSPSSGSN9_GS`\j"GSaSSPSSGSN9_GSb\j"GScSSPSSGSN9_GSd\j"GSeSSPSSSGS[9_GSf\j"GSgSSPSSSGS[9_GSh\j"S/S9_E0GSi\j"S/S9_GSj\j"S/S9_GSk\j"GSlSPGS'\\ SGSm9_GSn\j"GSoSPGS'\\ SGSm9_GSp\j"GSqSPGS'\\ SGSm9_GSr\j"GSsGS'\\ SGSt9_GSu\j"GSvGS'\\ SGSt9_GSw\j"GSxGS'SSY\\ SGSy9_GSz\j"GS{GS'\\ SGSt9_GS|\j"SPSSGS}9_GS~\j"SPSS9_GS\j"SPSS9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSXSGS}9_GS\j"SPSXSGS}9_E0GS\j"SPGSSGS}9_GS\j"SPGSSGS}9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSXSGS}9_GS\j"SPSXSGS}9_GS\j"SPGSSGS}9_GS\j"SPSS9_GS\j"SPSS9_GS\j"SPGS$SGS}9_GS\j"SPGS$SGS}9_GS\j"SPSXSGS}9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSXSGS}9_GS\j"SPGSSGS}9_GS\j"SPSSGS}9_E0GS\j"SPSXSGS}9_GS\j"SPSSGS}9_GS\j"SPSS9_GS\j"SPSS9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSXSGS}9_GS\j"SPSXSGS}9_GS\j"SPGSSGS}9_GS\j"SPGSSGS}9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSXSGS}9_GS\j"SPSXSGS}9_GS\j"SPGSSGS}9_GS\j"SPSS9_E0GS\j"SPSS9_GS\j"SPSS9_GS\j"SPSS9_GS\j"SPSS9_GS\j"SPSS9_GS\j"SPGS$SGS}9_GS\j"SPGS$SYSS9_GS\j"SPSXSYSS9_GS\j"SPSSYSS9_GS\j"SPSSYSS9_GS\j"SPSSYSS9_GS\j"SPSSYSS9_GS\j"SPSSYSS9_GS\j"SPSSYSS9_GS\j"SPSSYSS9_GS\j"SPSSYSS9_GS\j"SPGSSYSS9_E0GS\j"SPGSSYSS9_GS\j"SPGSSYSS9_GS\j"SPGSSYSS9_GS\j"SPGSSYSS9_GS\j"SPSSGS}9_GS\j"SPSSGS}9_GS\j"SPSXSGS}9_GS\j"SPGSSGS}9_GS\j"SPSSGS}9_GS\j"SPSXSGS}9_GS\j"SPGS$SYSS9_GS\j"SPGS$SYSS9_GS\j"SPGS\#\$GSGS9_GS\j"SPGS\#\$GSGS9_GS\j"SPGS\#\$GSGS9_GS\j"SPGS\#\$GSGS9_GS\j"SPSSS9_E0GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSSGS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSSGS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSSGS9_GS\j"SPSSSGS9_E0GS\j"SPSXSYS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSS9_GS\j"SPSSSGS9_GS\j"SPSSSGS9_GS\j"SPSXSYS9_GS\j"SGS9_GS\j"SPSSS9_GS\j"SPSSSGS9_GS\j"SPSXSYS9_GS\j"SGS9_GS\j"SPSSYS9_GS\j"SPSSSYGS9_GS\j"SPSXSYS9_GS\j"SPSSYSS9_E0GS\j"SPGS\\ SSYSGS9_GS\j"SP\\ SSYSGS9_GS\j"SP\#\$GSSYSGS9_GS\j"SP\#\$GSSYSGS9_GS\j"SP\#\$GSSYSGS9_GS\j"SP\#\$GSSYSGS9_GS\j"SP\#\$GSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SP\#\$GSSSYSGS9_GS\j"SPGSSS9_E\j"SPGSSS9\j"SPGSSS9\j"GSSS9\j"SP\\ SYS9\j"SP\\ SYS9\j"SP\\ SYS9\j"SP\\ SYS9\j"S/\\ SYGS9\j"S/S\\ SYGS9\j"SP\\ SYSGS9\j"SP\\ SYSGS9\j"SP\\ SYSGS9\j"SP\\ SYSGS9GS. Erk\kR5VVs/sH1upURGSS:5(dMGS UGSS;dM/UPM3 snnrn\nHBro\R"\k\o5rq\qGS SP:Xa SP\o-\qGS '\q\k\oRGS GS 5'MD \F"\k5rk\RRGS GS5R5GS:HruGSGS\YGS\OGS\ \OS\\[GS44GSjjrv\GGSGS\OS\[4GSjj5rw\GGSGS\OS\[4GSjj5rx\GGSGS\OS\[4GSjj5ry\GGSGS\OS\[4GSjj5rz\GGSGS\OS\[4GSjj5r{\GGSGS\OS\[4GSjj5r|\GGSGS\OS\[4GSjj5r}\GGSGS\OS\[4GSjj5r~\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GS jj5r\GGSGS\OS\[4GS!jj5r\GGSGS\OS\[4GS"jj5r\GGSGS\OS\[4GS#jj5r\GGSGS\OS\[4GS$jj5r\GGSGS\OS\[4GS%jj5r\GGSGS\OS\[4GS&jj5r\GGSGS\OS\[4GS'jj5r\GGSGS\OS\[4GS(jj5r\GGSGS\OS\[4GS)jj5r\GGSGS\OS\[4GS*jj5r\GGSGS\OS\[4GS+jj5r\GGSGS\OS\[4GS,jj5r\GGSGS\OS\[4GS-jj5r\GGSGS\OS\[4GS.jj5r\GGSGS\OS\[4GS/jj5r\GGSGS\OS\[4GS0jj5r\GGSGS\OS\[4GS1jj5r\GGSGS\OS\[4GS2jj5r\GGSGS\OS\[4GS3jj5r\GGSGS\OS\[4GS4jj5r\GGSGS\OS\[4GS5jj5r\GGSGS\OS\[4GS6jj5r\GGSGS\OS\[4GS7jj5r\GGSGS\OS\[4GS8jj5r\GGSGS\OS\[4GS9jj5r\GGSGS\OS\[4GS:jj5r\GGSGS\OS\[4GS;jj5r\GGSGS\OS\[4GS<jj5r\GGSGS\OS\[4GS=jj5r\GGSGS\OS\[4GS>jj5r\GGSGS\OS\[4GS?jj5r\GGSGS\OS\[4GS@jj5r\GGSGS\OS\[4GSAjj5r\GGSGS\OS\[4GSBjj5r\GGSGS\OS\[4GSCjj5r\GGSGS\OS\[4GSDjj5r\GGSGS\OS\[4GSEjj5r\GGSGS\OS\[4GSFjj5r\GGSGS\OS\[4GSGjj5r\GGSGS\OS\[4GSHjj5r\GGSGS\OS\[4GSIjj5r\GGSGS\OS\[4GSJjj5r\GGSGS\OS\[4GSKjj5r\GGSGS\OS\[4GSLjj5r\GGSGS\OS\[4GSMjj5r\GGSGS\OS\[4GSNjj5r\GGSGS\OS\[4GSOjj5r\GGSGS\OS\[4GSPjj5r\GGSGS\OS\[4GSQjj5r\GGSGS\OS\[4GSRjj5r\GGSGS\OS\[4GSSjj5r\GGSGS\OS\[4GSTjj5r\GGSGS\OS\[4GSUjj5r\GGSGS\OS\[4GSVjj5r\GGSGS\OS\[4GSWjj5r\GGSGS\OS\[4GSXjj5r\GGSGS\OS\[4GSYjj5r\GGSGS\OS\[4GSZjj5r\GGSGS\OS\[4GS[jj5r\GGSGS\OS\[4GS\jj5r\GGSGS\OS\[4GS]jj5r\GGSGS\OS\[4GS^jj5r\GGSGS\OS\[4GS_jj5r\GGSGS\OS\[4GS`jj5r\GGSGS\OS\[4GSajj5r\GGSGS\OS\[4GSbjj5r\GGSGS\OS\[4GScjj5r\GGSGS\OS\[4GSdjj5r\GGSGS\OS\[4GSejj5r\GGSGS\OS\[4GSfjj5r\GGSGS\OS\[4GSgjj5r\GGSGS\OS\[4GShjj5r\GGSGS\OS\[4GSijj5r\GGSGS\OS\[4GSjjj5r\GGSGS\OS\[4GSkjj5r\GGSGS\OS\[4GSljj5r\GGSGS\OS\[4GSmjj5r\GGSGS\OS\[4GSnjj5r\GGSGS\OS\[4GSojj5r\GGSGS\OS\[4GSpjj5r\GGSGS\OS\[4GSqjj5r\GGSGS\OS\[4GSrjj5r\GGSGS\OS\[4GSsjj5r\GGSGS\OS\[4GStjj5r\GGSGS\OS\[4GSujj5r\GGSGS\OS\[4GSvjj5r\GGSGS\OS\[4GSwjj5r\GGSGS\OS\[4GSxjj5r\GGSGS\OS\[4GSyjj5r\GGSGS\OS\[4GSzjj5r\GGSGS\OS\[4GS{jj5r\GGSGS\OS\[4GS|jj5r\GGSGS\OS\[4GS}jj5r\GGSGS\OS\[4GS~jj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5r\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr \GGSGS\OS\[4GSjj5Gr \GGSGS\OS\[4GSjj5Gr \GGSGS\OS\[4GSjj5Gr \GGSGS\OS\[4GSjj5Gr \GGSGS\OS\[4GSjj5Gr\GGSGS\OS\[4GSjj5Gr\H"\K0GSS_GSS_GSS_GSS_GSS_GSS_GSS_GSS_GSS_GSGS_GSGS_GSS_GSS_GSS_GSS_GSS_GSGS_GSGS GS GS.E5 g!\a SSKJr GNf=fs snnf(aVision Transformer (ViT) in PyTorch A PyTorch implement of Vision Transformers as described in: 'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale' - https://arxiv.org/abs/2010.11929 `How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers` - https://arxiv.org/abs/2106.10270 `FlexiViT: One Model for All Patch Sizes` - https://arxiv.org/abs/2212.08013 The official jax code is released and available at * https://github.com/google-research/vision_transformer * https://github.com/google-research/big_vision Acknowledgments: * The paper authors for releasing code and weights, thanks! * I fixed my class token impl based on Phil Wang's https://github.com/lucidrains/vit-pytorch * Simple transformer style inspired by Andrej Karpathy's https://github.com/karpathy/minGPT * Bert reference code checks against Huggingface Transformers and Tensorflow Bert Hacked together by / Copyright 2020, Ross Wightman N) OrderedDict)partial) AnyCallableDictOptionalSetTupleTypeUnionList)Literal)Final)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STDOPENAI_CLIP_MEANOPENAI_CLIP_STD) Attention DiffAttentionAttentionPoolLatent PatchEmbedMlp SwiGLUPackedSwiGLU LayerNormRmsNormDropPathcalculate_drop_path_rates PatchDropout trunc_normal_ lecun_normal_resample_patch_embedresample_abs_pos_embeduse_fused_attn get_act_layerget_norm_layermaybe_add_mask LayerType LayerScale)build_model_with_cfg)feature_take_indices) named_apply checkpointcheckpoint_seqadapt_input_conv)generate_default_cfgsregister_modelregister_model_deprecationsVisionTransformer)attndiffT attn_layerdim num_headsqkv_biasqk_norm scale_norm proj_bias attn_drop proj_drop norm_layerdepthreturnc [U[5(a#[RUS5nUc SU35e[ U[ 5(aXS'U"U4UUUUUUUU S.U D6$)NzUnknown attn_layer: rD)r<r=r>r?r@rArBrC) isinstancestr ATTN_LAYERSget issubclassr) r:r;r<r=r>r?r@rArBrCrDkwargss ]/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/vision_transformer.py _create_attnrNZs*c"" __Z6 %J)=j\'JJ%*m,,w       c%b^\rSrSrSrSSSSSSSSSS\R \\\ SSS4S \ S \ S \ S \ S \ S\ S\ S\ S\ S\ S\ \ S\ S\\RS\\RS\\RS\S\ SS4$U4SjjjrS S\R&S\ \R&S\R&4SjjrSrU=r$)!Block~z)Transformer block with pre-normalization.@FTNrr;r< mlp_ratior=r>scale_attn_normscale_mlp_normr@rBrA init_values drop_path act_layerrC mlp_layerr:rDrEc `>[TU]5 UUS.nU"U40UD6Ul[UU4UUUUUU U UUS. UD6UlU (a[ U4SU 0UD6O[ R"5UlU S:a [U 5O[ R"5Ul U"U40UD6Ul U"SU[X-5U U(aUOSUU S.UD6Ul U (a[ U4SU 0UD6O[ R"5UlU S:a[U 5Ulg[ R"5Ulg)aInitialize Block. Args: dim: Number of input channels. num_heads: Number of attention heads. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. qk_norm: If True, apply normalization to query and key. proj_bias: If True, add bias to output projection. proj_drop: Projection dropout rate. attn_drop: Attention dropout rate. init_values: Initial values for layer scale. drop_path: Stochastic depth rate. act_layer: Activation layer. norm_layer: Normalization layer. mlp_layer: MLP layer. attn_layer: Attention layer type (class or string). depth: Block index, passed to attention layer for depth-dependent init. devicedtype r<r=r>r?r@rArBrCrDrXrTN in_featureshidden_featuresrZrCbiasdrop)super__init__norm1rNr8r+nnIdentityls1r drop_path1norm2intmlpls2 drop_path2selfr;r<rUr=r>rVrWr@rBrArXrYrZrCr[r:rDr^r_dd __class__s rMrhBlock.__init__s;R  /*r*      &!     FQ:cA{AbAVXVaVaVc1:R(9-R[[]*r*  0%3z   FQ:cA{AbAVXVaVaVc1:R(9-R[[]rOx attn_maskc XRURURURU5US955-nXR UR UR URU5555-nU$Nry)rmrlr8rirrrqrprnrtrxrys rMforward Block.forwardsa 4::a=I)V WX X $**Q-)@ AB BrO)r8rmrrrlrqrprirnN__name__ __module__ __qualname____firstlineno____doc__rjGELUrrrrofloatboolrr Moduler*rhtorchTensorr~__static_attributes__ __classcell__rvs@rMrQrQ~ss3 ""!$)#("!!+/!)+*3),$-)ISISIS IS  IS  IS"IS!ISISISIS"%ISISBIIISRYYIS BII!IS""#IS$%IS* +ISISV(5<<2HTYT`T`rOrQc%h^\rSrSrSSSSSSSSSS\R \\\SSS4S\ S \ S \ S \ S \ S \ S\ S\ S\ S\ S\ \ S\ S\ \RS\ \RS\ \RS\S\ SS4$U4SjjjrS SjrS!S\R&S\ \R&S\R&4SjjrSrU=r$)" ResPostBlockrSFTrTNrr;r<rUr=r>rVrWr@rBrArXrYrZrCr[r:rDrEc >[TU]5 UUS.nXl[UU4UUUUUU U UUS. UD6UlU"U40UD6UlU S:a [ U 5O[R"5Ul U"SU[X-5U U(aUOSUU S.UD6Ul U"U40UD6Ul U S:a [ U 5O[R"5Ul UR5 g)Nr]r`rTrarf)rgrhrXrNr8rirrjrkrmrorprnrr init_weightsrss rMrhResPostBlock.__init__s,  /&     &!      *r* 1:R(9-R[[] 0%3z    *r* 1:R(9-R[[] rOcURb}[RRURR UR5 [RRUR R UR5 ggr)rXrjinit constant_riweightrnrts rMrResPostBlock.init_weights s[    ' GG  djj//1A1A B GG  djj//1A1A B (rOrxryc XRURURXS955-nXRUR UR U555-nU$r{)rmrir8rrrnrpr}s rMr~ResPostBlock.forwardsN  499Q9+L MN N  488A; 78 8rO)r8rmrrrXrprirnrENr)rrrrrjrrrrrorrrr rr*rhrrrr~rrrs@rMrrsN ""!$)#("!!+/!)+*3),$-)777 7  7  7"7!7777"%77BII7RYY7 BII!7""#7$%7* +77rC (5<<2HTYT`T`rOrc'^\rSrSr%Sr\\\S'SSSSSSSSSS\R\ SSS SSS4S \ S \ S \ S \S\S\S\S\S\ S\ S\ \ S\ S\\RS\\RS\ \\RS\ \S\ S\SS4&U4SjjjrS#SjrS$S\R(S \ \R(S\R(4S!jjrS"rU=r$)%ParallelScalingBlockizParallel ViT block (MLP & Attention in parallel) Based on: 'Scaling Vision Transformers to 22 Billion Parameters` - https://arxiv.org/abs/2302.05442 fused_attnrSFTrTNrr;r<rUr=r>rVrWr@rBrArXrYrZrCr[r:rD fuse_out_projrEc&>[TU]5 UUS.nX-S:XdS5eU(dU(aS5eX lX-UlURS-Ul[ 5Ul[X1-5nUSU--nU"U40UD6Ul[R"UU4SU0UD6Ul U/U/S--Ul U(aURSS5 O0[R"[R "U40UD65UlU(aU"UR40UD6O[R$"5UlU(aU"UR40UD6O[R$"5Ul[R*"U 5Ul[R*"U 5UlU "5UlU(a1[R"UU-U4SU0UD6UlSUlSUlODSUl[R"X4SU0UD6Ul[R"UU4SU0UD6UlU b[9U4S U 0UD6O[R$"5UlU S :a [=U 5O[R$"5UlURA5 g) Nr]r$dim should be divisible by num_headsScale norms not supportedrdmlp_biasrXrT)!rgrhr<head_dimscaler&rroin_normrjLinearin_projin_splitregister_parameter Parameterremptyrrkq_normk_normDropoutrAmlp_dropmlp_actout_proj attn_out_proj mlp_out_projr+lsrrYreset_parameters)rtr;r<rUr=r>rVrWr@rBrArXrYrZrCr[r:rDrr^r_rumlp_hidden_dimin_proj_out_dimrvs rMrhParallelScalingBlock.__init__s$.  /!#K%KK#">V;VV9"( ]]d* (*Y_-(1s72!#,, yyoKHKK '(C5194   # #J 5LL^)Jr)JKDM9@j5"5bkkm 9@j5"5bkkm I. 9-  { IIcN&:CViVSUVDM!%D  $D !DM!#3!J)!Jr!JD  " .# TI TQS TD DOD[*S@k@R@acalalan09B),BKKM rOcrURb*[RRUR5 gg)"Initialize parameters and buffers.N)rrjrzeros_rs rMr%ParallelScalingBlock.reset_parametersbs% == $ GGNN4== ) %rOrxryc"URup4nURU5nURU5n[R"X`R SS9upxpUR bXpR -nURURX4URUR55RSS5nURU RX4URUR55RSS5n U RX4URUR5RSS5n UR(a?[R"XU UUR (aUR"R$OSS9n OQXR&-nXRSS5-n [)X5n U R+SS9n UR#U 5n X-n U RSS5R-X4U5n UR/U5nUR1U5nUR2b&UR3[R4"X4SS95nO#UR7U 5UR9U5-nXR;UR=U55-nU$)Nr;r,rTry dropout_p)shaperrrsplitrrrviewr<r transposerrFscaled_dot_product_attentiontrainingrAprr)softmaxreshaperrrcatrrrYr) rtrxryBNCyx_mlpqkvx_attnr8s rMr~ParallelScalingBlock.forwardgs ''a LLO LLOQ 2>! == $MM)E KKqT^^T]]C D N NqRS T KKqT^^T]]C D N NqRS T FF1 7 A A!Q G ??33a#.2mm$..**F JJA{{2r**D!$2D<>$'DXF!!!Q'//a8 U# e$ == $ eiiR@AA""6*T->->u-EEA twwqz* *rO)rArrYrrrrrrrrrrrr<rrrrr)rrrrrrr__annotations__rjrrrorrr rr*rhrrrr~rrrs@rMrrsd  ""!$)#("!!+/!)+*337.2"'+A A A  A  A  A "A !A A A A "%A A BIIA RYYA  RYY0!A "!+#A $%A & 'A , -A A F* **(5<<2H*TYT`T`**rOrc'^\rSrSr%Sr\\\S'SSSSSSSSSS\R\ SSS SSS4S \ S \ S \ S \S\S\S\S\S\ S\ S\ \ S\ S\\RS\\RS\ \\RS\ \S\ S\SS4&U4SjjjrS\ 4SjrS%SjrS\R*4S jrS&S!\R*S"\ \R*S\R*4S#jjrS$rU=r$)'DiffParallelScalingBlockia<Parallel ViT block with Differential Attention (MLP & Attention in parallel). Combines the parallel MLP+Attention structure from 'Scaling Vision Transformers to 22 Billion Parameters' (https://arxiv.org/abs/2302.05442) with differential attention from 'Differential Transformer' (https://arxiv.org/abs/2410.05258). rrSFTrTNrr;r<rUr=r>rVrWr@rBrArXrYrZrCr[r:rD dual_lambdarEc>[TU]5 UUS.nX-S:XdS5eU(dU(aS5eX lX-S-UlURS-Ul[ 5Ul[X1-5nUSU--nU"U40UD6Ul[R"UU4SU0UD6Ul U/U/S--Ul U(aURS S5 O0[R"[R "U40UD65UlU(aU"UR40UD6O[R$"5UlU(aU"UR40UD6O[R$"5Ul[R*"U 5UlXl[1SUR-4S S 0UD6UlUUlU(a[R"[R "S [R6US 95Ul[R"[R "S [R6US 95UlS=Ul=Ul=Ul Ul!GO*S=UlUl[R"[R "UR[R6US 95Ul[R"[R "UR[R6US 95Ul[R"[R "UR[R6US 95Ul [R"[R "UR[R6US 95Ul![R*"U 5Ul"U "5Ul#[R"UU-U4SU0UD6Ul$U b[KU4SU 0UD6O[R$"5Ul&U S:a [OU 5O[R$"5Ul(SUl)URUU5 URW5 g)Nr]rrrrrrrdrepsh㈵>rf)r_r^rXrT皙?),rgrhr<rrr&rrorrjrrrrrrrrrkrrrrA attn_drop_prsub_normrfloat32lambda_alambda_b lambda_q1 lambda_k1 lambda_q2 lambda_k2rrrr+rrrY lambda_initset_lambda_initr)rtr;r<rUr=r>rVrWr@rBrArXrYrZrCr[r:rDrr^r_rurrrvs rMrh!DiffParallelScalingBlock.__init__s#.  /!#K%KK#">V;VV9"(A- ]]d* (*Y_-(1s72!#,, yyoKHKK '(C5194   # #J 5LL^)Jr)JKDM9@j5"5bkkm 9@j5"5bkkm I.$ DMM 1BtBrB & LLRu}}U[)\]DMLLRu}}U[)\]DMPT TDN TT^ Tdnt~,0 0DMDM\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN 9-  {  #"6R)RrR DOD[*S@k@R@acalalan09B),BKKM U# rOcLSS[R"SU-5-- Ulg)Nrg333333?g333333ӿ)mathexpr)rtrDs rMr(DiffParallelScalingBlock.set_lambda_inits!txxu '=!==rOcURb)[RRUR5 UR(aS[RRUR 5 [RRUR 5 g[RRURSSS9 [RRURSSS9 [RRURSSS9 [RRURSSS9 g)rNr皙?meanstd) rrjrrrrrnormal_rrrrrs rMr)DiffParallelScalingBlock.reset_parameterss == $ GGNN4== )    GGNN4== ) GGNN4== ) GGOODNNO < GGOODNNO < GGOODNNO < GGOODNNO >DNN+JPR!S!Y!Y![\Hyy4>>DNN+JPR!S!Y!Y![\H"T%5%555rOrxrycURup4nURU5nURU5n[R"X`R SS9upxpUR bXpR -nURX4SUR-UR5RSS5nU RX4SUR-UR5RSS5n U RX4URSUR-5RSS5n URU5URU 5pUR5RU5n UR(aURX0RSX@R5nU RX0RSX@R5n UR!S5upU R!S5upUR"(a UR$OSn[&R("XXUS9n[&R("XXUS9nUU U-- nOXR*-nXRSS5-n[-UU5nUR/SS9nUR1U5nUR3X0RSXD5nUSS2SS2S4U USS2SS2S4-- nUU -nUR5U5nUSUR6- -nURSS5RX4U5nUR9U5nUR;U5nUR=[R>"UU4SS95nXRAURCU55-nU$) Nrrrr,rTrrr)"rrrrrrrrr<rrrrrtype_asrunbindrrrrrr)rrArrrrrrrrYr)rtrxryrrrrrrrr lambda_fullq1q2k1k2rattn1attn2rr8s rMr~ DiffParallelScalingBlock.forwards''a LLO LLOQ 2>! == $MM)E IIaA. > H HA N IIaA. > H HA N IIaDNNA ,= > H HA N{{1~t{{1~1**,44Q7 ?? !^^Q==AA !^^Q==AAXXa[FBXXa[FB,0MM((sI2221]fgE2221]fgE[500FJJA{{2r**D!$ 2D<>$'D99Q18D1a=;aAg#>>DAXFv&1t///0!!!Q'//a8 U# e$ MM%))VUO< = twwqz* *rO)rArrYrrrrrrrrrrrrrrrrrrr<rrrrrr)rrrrrrrrrjrrrorrr rr*rhrrrrrr~rrrs@rMrrs d  ""!$)#("!!+/!)+*337.2 %+K K K  K  K  K "K !K K K K "%K K BIIK RYYK  RYY0!K "!+#K $%K &'K , -K K Z>S> =6666(5<<2H6TYT`T`66rOrc'h^\rSrSrSrSSSSSSSSSSS\R \\\ S SS4S \ S \ S \ S \ S\ S\ S\ S\ S\ S\ \ S\ S\ S\ S\\RS\\RS\\RS\S\ SS4&U4SjjjrS"S\R&S\ \R&S\R&4S jjrS!rU=r$)#ParallelThingsBlocki<zParallel ViT block (N parallel attention followed by N parallel MLP) Based on: `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795 rrSFTNrTrr;r< num_parallelrUr=r>rVrWr@rXrBrArYrZrCr[r:rDrEc>UUS.n[TU]5 X0l[R"5Ul[R"5Ul[U5GHhnUR R[R"[SU"U40UD64S[UU4UUUUU U U UUS. UD64SU (a[U4SU 0UD6O[R"54SU S:a [U 5O[R"54/555 UR R[R"[SU"U40UD64S U"U4[X-5UU(aUOSU U S .UD64SU (a[U4SU 0UD6O[R"54SU S:a [U 5O[R"54/555 GMk g) Nr]normr8r`rrXrYrTrp)rcrZrCrdre)rgrhrrj ModuleListattnsffnsrangeappend SequentialrrNr+rkrro)rtr;r<rrUr=r>rVrWr@rXrBrArYrZrCr[r:rDr^r_ru_rvs rMrhParallelThingsBlock.__init__As. / (]]_ MMO |$A JJ  bmmKC.2./ (%#.''')   z#E;E"EZ\ZeZeZghY^hy1W#9- & II  R]];C.2./ $'$8'-;z"" z#E;E"EZ\ZeZeZghY^hy1W 8 ,  )%rOrxryc VUb/nURHVnURU5nURXRS9nURU5nUR U5nUR U5 MX U[ R"U5RSS9-nOFU[ R"URVs/sH oD"U5PM sn5RSS9-nU[ R"URVs/sH of"U5PM sn5RSS9-nU$s snfs snf)Nr|rr) rrr8rrYrrstackrr)rtrxryattn_outr8rffns rMr~ParallelThingsBlock.forwards  H 16?/' # EKK)--!-44AEKKTZZ @ZTaZ @AEE!ELLA  tyy9ySVy9:>>1>E E!A9s 2D! 8D& )rrrrrrs@rMrr<sZ!"!"!$)#("+/!!!)+*3),$-+=== =  =  =="=!=="%====BII= RYY!="BII#=$"%=&'=, -==~  (5<<2H TYT`T`  rOrrx pool_typenum_prefix_tokensreduce_include_prefixc4U(dU$US:Xa USS2S4nU$U(aUO USS2US24nUS:XaURSS9nU$US:Xa$SURSS9URSS9--nU$US:XaURSS9nU$U(a S U35eU$) Ntokenravgr,ravgmaxg?maxzUnknown pool type )ramax)rxr!r"r#s rMglobal_pool_nlcr*s G adG H'AAa1B1C.C,D  1 A H( "qvv!v}qvv!v}45A H % 1 A H! B$6yk"B B= HrOcS:^\rSrSr%Sr\\\S'SSSSSS S S S S S S S S SS SS SS S SS S S SSSSSSSS \SSS\ \ \ SS4*S\ \ \\ \ 44S\ \ \\ \ 44S\ S\ S\SS\ S\ S\ S\S\S\S\S \S!\S"\\S#\S$\S%\S&\ S'\S(\S)\\S*\S\S+\S,\S-\S.\S/\S0\S1\S2\S3S4\S5\S6\\S7\\S8\\S9\\R.S:\\R.S;\SjrShS?\S@\SrVrWr@rX class_token pos_embedno_embed_class reg_tokenspre_norm final_normfc_normpool_include_prefixdynamic_img_pad drop_rate pos_drop_ratepatch_drop_rateproj_drop_rateattn_drop_ratedrop_path_rate weight_init)skipresetjaxjax_nlhbmocor7fix_init embed_layerembed_norm_layerrCrZblock_fnr[r:rEc+ P >[T3U]5 U)U*S.n+US;deU(dUS:wdeUS;deUcUS;OUn,[U$5=(d [n$[U#5n#[ U%5=(d [ R n%X@lX0lXPl U=Ul =Ul Ul U(aSOSUl U=RU- sl UUlUUlUUlUUlUUlS Ul0n-U(aU-R)[+S S S 95 U#bU#U-S 'U""S)UUUUU(+US .U-DU+D6UlUR,R.n.[1UR,S5(aUR,R35OUn/U(a-[ R4"[6R8"SSU40U+D65OSUlU(a-[ R4"[6R8"SUU40U+D65OSUlU(aU.OU.UR-n0U(aUS:XaSUlO2[ R4"[6R8"SU0U40U+D65Ul[ R@"US9Ul!US:a[EUURS9Ul#O[ RH"5Ul#U(a U$"U40U+D6O[ RH"5Ul%[MUU5n1[ RN"[QU5V2s/sHAn2U&"S)0SU_SU_SU _SU _SU _SU _SU _SU_SU_SU_SU_SU1U2_S U$_SU%_SU'_S U(_S!U2_U+D6PMC sn26Ul)[QU5V2s/sHn2[+S"U23UU/S#9PM sn2Ul*U(aU,(d U$"U40U+D6O[ RH"5Ul+US$:Xa![YUR4UU U$U%S%.U+D6Ul-OSUl-U(aU,(a U$"U40U+D6O[ RH"5Ul.[ R@"U5Ul/US:a"[ R`"URU40U+D6O[ RH"5Ul1U S&:XaS'OU Ul2U!Ul3U S&:waURiS S(9 ggs sn2fs sn2f)*a Args: img_size: Input image size. patch_size: Patch size. in_chans: Number of image input channels. num_classes: Number of classes for classification head. global_pool: Type of global pooling for final sequence (default: 'token'). embed_dim: Transformer embedding dimension. depth: Depth of transformer. num_heads: Number of attention heads. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: Enable bias for qkv projections if True. init_values: Layer-scale init values (layer-scale enabled if not None). class_token: Use class token. no_embed_class: Don't include position embeddings for class (or reg) tokens. reg_tokens: Number of register tokens. pre_norm: Enable norm after embeddings, before transformer blocks (standard in CLIP ViT). final_norm: Enable norm after transformer blocks, before head (standard in most ViT). fc_norm: Move final norm after pool (instead of before), if None, enabled when global_pool == 'avg'. drop_rate: Head dropout rate. pos_drop_rate: Position embedding dropout rate. attn_drop_rate: Attention dropout rate. drop_path_rate: Stochastic depth rate. weight_init: Weight initialization scheme. fix_init: Apply weight initialization fix (scaling w/ layer index). embed_layer: Patch embedding layer. embed_norm_layer: Normalization layer to use / override in patch embed module. norm_layer: Normalization layer. act_layer: MLP activation layer. block_fn: Transformer block layer. r]r8r%)r7noner2N)r&r'r(r,rFNHWC)strict_img_size output_fmtrC)r3r4r5r:rdrC feat_ratiorU)r)r"r;r<rUr=r>rVrWr@rXrBrArYrZr[r:rDblocks.)modulenum_chs reductionr9)r<rUrCrZrKrL needs_resetrf)5rgrhr(rr'rjrr6r5r7 num_featureshead_hidden_sizer:r"num_reg_tokenshas_class_tokenr=rBr,grad_checkpointingupdatedict patch_embed num_patcheshasattrrYrrr cls_token reg_tokenr<rpos_dropr! patch_droprknorm_prer rrblocks feature_inforr attn_poolrA head_droprheadweight_init_moderPr)4rtr3r4r5r6r7r:rDr<rUr=r>rVrWr@rXr;r<r=r>r?r@rArBr,rCrDrErFrGrHrIrJrPrQrRrCrZrSr[r:r^r_ru use_fc_norm embed_argsrhr] embed_lendprirvs4 rMrhVisionTransformer.__init__sX  /JJJJkW4441111AHk%==V] #J/<9 )*:;!),7 & &ENNND1DN&1q *,(*,#6 0"'    d5VL M  ''7J| $&  !+      &&22 5mm*5\+&#*")   # $ "      !0   . $ ( ) ) a& & $ $  &! "% ("+&#$ .Y^^cXdfXdSTD'! yI NXdf3=kJy/B/WYWbWbWd  % 0##%#   DN"DN6@[z)2r2VXVaVaVc I.DORSOBIIdnnk@R@Y[YdYdYf +6&+@k  &   %  0 ![&#,fs AR$R#c[R"5 [UR5Hup[R "SUS--5nUR RRRU5 URRRRU5 M SSS5 g!,(df  g=f)z9Apply weight initialization fix (scaling w/ layer index).g@r,N) rno_grad enumeraterorsqrtr8projrdiv_rpfc2)rtlayer_idlayerrs rMfix_init_weight!VisionTransformer.fix_init_weightisy ]]_#,T[[#9 #A"67 &&++E2 $$))%0$:__s BB88 Cmoder_cU=(d URnUS;deSU;a![R"UR5*OSnURb[ URSS9 UR b([RRUR SS9 URb([RRURSS9 [[XUS9U5 UR(aUR5 gg) aInitialize model weights. Args: mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or ''). needs_reset: If True, call reset_parameters() on modules that have it. Set to False when modules have already self-initialized in __init__. )rMrNrOrLr7nlhbrTN{Gz?rư>r^)rtrlogr6r<r"rjrjrrrkr/get_init_weights_vitrPr)rtrr_ head_biass rMrVisionTransformer.init_weightsqs,t,,????39T>TXXd..//r >> % $..c 2 >> % GGOODNNO 5 >> % GGOODNNO 5(kRTXY ==  " rOmc[U5 g)z>Initialize weights for a single module (compatibility method).N)init_weights_vit_timm)rtrs rM _init_weightsVisionTransformer._init_weightss  a rOcheckpoint_pathprefixc[XU5 g)zqLoad pretrained weights. Args: checkpoint_path: Path to checkpoint. prefix: Prefix for state dict keys. N) _load_weights)rtrrs rMload_pretrained!VisionTransformer.load_pretraineds dV4rOc 1Sk$)z3Set of parameters that should not use weight decay.>rjr< dist_tokenrfrs rMno_weight_decay!VisionTransformer.no_weight_decays 87rOcoarsec[SSS/S9$)zCreate regex patterns for parameter grouping. Args: coarse: Use coarse grouping. Returns: Dictionary mapping group names to regex patterns. z ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i)stemro)rf)rtrs rM group_matcherVisionTransformer.group_matchers4-/CD  rOenablec~Xl[URS5(aURRU5 gg)zgEnable or disable gradient checkpointing. Args: enable: Whether to enable gradient checkpointing. set_grad_checkpointingN)rdrirgr)rtrs rMr(VisionTransformer.set_grad_checkpointings7#) 4##%= > >    3 3F ; ?rOcUR$)zGet the classifier head.)rsrs rMget_classifier VisionTransformer.get_classifiersyyrOc(XlUbBUS;deUS:XaURcS5eUS:waURbSUlX lUS:a'[R"UR U5Ulg[R "5Ulg)zReset the classifier head. Args: num_classes: Number of classes for new classifier. global_pool: Global pooling type. Nr8r9z=Cannot currently add attention pooling in reset_classifier().r)r6rqr7rjrr:rkrs)rtr6r7s rMreset_classifier"VisionTransformer.reset_classifiers'  ""NN NNe#(>]]]u%$..*D!%* >IAoBIIdnnk: SUS^S^S` rOc URRnURRXS9 URbUR(aSO UR nURR U-nXPRRS:waE[R"[URURRUUSS95Ulggg)zUpdate the input image resolution and patch size. Args: img_size: New input resolution, if None current resolution is used. patch_size: New patch size, if None existing patch size is used. )r3r4Nrr,T)new_sizeold_sizer"verbose) rg grid_sizeset_input_sizer<r=r"rhrrjrr%)rtr3r4prev_grid_sizer"num_new_tokenss rMr VisionTransformer.set_input_sizes))33 '''Q >> %%)%8%8d>T>T !--99> !66!''!*b!''"+6 6  JA!!--77N.''+':':!@V@V I q"a AI >> % MM$..// BC D >> % MM$..// BC D    AIIfsl2IIfsl2 A}}QrOindicesreturn_prefix_tokensr stop_earlyrXintermediates_only output_dictryc US;dS5eUS:Hn /n [[UR5U5upURupnnUR U5nUR U5nUR U5nURU5n[RR5(dU(d URnOURSU S-n[U5HunnU bU"XS9nOIUR(a0[RR5(d [UU5nOU"U5nUU ;dMaU RU(aURU5OU5 M UR (aKU Vs/sHnUSS2SUR 24PM nnU Vs/sHnUSS2UR S24PM n nOSnU (adURR#UU45unnU Vs/sH7nUR%UUUS5R'SS SS 5R)5PM9 n nU(a50nU US 'Ub U(aUUS 'U(dURU5nUUS 'U$[RR5(dU(aUb[+[-U U55n U(aU $URU5nX4$s snfs snfs snf)ajForward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence return_prefix_tokens: Return both prefix and spatial intermediate tokens norm: Apply norm layer to all intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features output_dict: Return outputs as a dictionary with 'image_features' and 'image_intermediates' keys attn_mask: Optional attention mask for masked attention (e.g., for NaFlex) Returns: A tuple with (final_features, intermediates), a list of intermediate features, or a dictionary containing 'image_features' and 'image_intermediates' (and optionally 'image_intermediates_prefix') )NCHWNLCz)Output format must be one of NCHW or NLC.rNr,r|rrrrimage_intermediatesimage_intermediates_prefiximage_features)r.lenrorrgrrmrnrjit is_scriptingr}rdr0rrr"dynamic_feat_sizerpermute contiguouslistzip)rtrxrrrrrXrrryr intermediates take_indices max_indexrrheightwidthroryblkr prefix_tokensrr result_dictx_finals rMforward_intermediates'VisionTransformer.forward_intermediates s8_,Y.YY,& "6s4;;7G"Q  ggfe   Q  OOA  OOA  MM!  99 ! ! # #:[[F[[)a-0F'FAs$/((1G1G1I1IsA&FL $$TTYYq\qA(  ! !ERS]Qq!D$:$:"::;]MSDQRMqQq$"8"8"99:MMRM M ##55vuoFDAq^kl^kYZQYYq!Q3;;Aq!QGRRT^kMl K1>K- .(-AK prune_norm prune_headc[[UR5U5upEURSUS-UlU(a[R"5UlU(a,[R"5UlURSS5 U$)a Prune layers not required for specified intermediates. Args: indices: Indices of intermediate layers to keep. prune_norm: Whether to prune normalization layer. prune_head: Whether to prune the classifier head. Returns: List of indices that were kept. Nr,rr7)r.rrorjrkrrAr)rtrrrrrs rMprune_intermediate_layers+VisionTransformer.prune_intermediate_layersksh #7s4;;7G"Q kk.9q=1  DI ;;=DL  ! !!R (rOnrc <URXUUU(aSOSSUS9$)axGet intermediate layer outputs (DINO interface compatibility). NOTE: This API is for backwards compat, favour using forward_intermediates() directly. Args: x: Input tensor. n: Number or indices of layers. reshape: Reshape to NCHW format. return_prefix_tokens: Return prefix tokens. norm: Apply normalization. Returns: List of intermediate features. rrT)rrrXrry)r)rtrxrrrrrys rMget_intermediate_layers)VisionTransformer.get_intermediate_layerss2.)) !5!(ve# *  rOcURU5nURU5nURU5nURU5nUbURH nU"XS9nM O\UR (a:[ RR5(d[URU5nOUR U5nURU5nU$)z\Forward pass through feature layers (embeddings, transformer blocks, post-transformer norm).r|) rgrrmrnrordrrrr1r)rtrxryrs rMforward_features"VisionTransformer.forward_featuress   Q  OOA  OOA  MM!   {{/#  $ $UYY-C-C-E-Et{{A.A AA IIaLrOr!cURb9UR(dUSS2URS24nURU5nU$Uc UROUn[ UUURURS9nU$)zApply pooling to feature tokens. Args: x: Feature tensor. pool_type: Pooling type override. Returns: Pooled features. N)r!r"r#)rqrBr"r7r*)rtrxr!s rMpoolVisionTransformer.pools} >> %++a//001q!AH(1(9D$$y  "44"&":":   rO pre_logitscURU5nURU5nURU5nU(aU$URU5$)zForward pass through classifier head. Args: x: Feature tensor. pre_logits: Return features before final classifier. Returns: Output tensor. )rrArrrs)rtrxrs rM forward_headVisionTransformer.forward_headsA IIaL LLO NN1 q0DIIaL0rOcFURXS9nURU5nU$r{)rrr}s rMr~VisionTransformer.forwards*  ! !! ! 9   a rO)rqrorjr,r:rArprPr7rdrcrsrrrar5r=rrnr6r`r"rbrmrgrBrlr<rkrtrr7Tr7F)Tr)NN)NFFFrFFN)r,FT)r,FFFN)5rrrrrrrrrrQrrr ror rrrrHrr*r rjrrhrrrrrignorerr rrr rrrrrrrrrrrrrrr~rrrs@rMr6r6s Dk!5868#OV !!!$)#("+/ $$#("#&*(-%*$)!#%%'$&$&$&SU"$.48.2-1(-),$-Wv1CsCx01v1c5c?23v1 v1  v1 !!KL v1v1v1v1v1v1v1"v1!v1v1 "%!v1"#v1$%v1&!'v1()v1*+v1,-v1.d^/v10"&1v12#3v14"5v167v18!9v1:#;v1<"=v1>"?v1@"Av1B!!OPCv1DEv1F"Gv1H'y1Iv1J!+Kv1L *Mv1N299oOv1PBIIQv1R"Sv1X Yv1v1p1####.!ryy!T!  YY5s5C555 YY8S88 YY  D  T#uS$Y?O:O5P     YY*  c 845!).04 || S$s)U3Z/0   #'      -  ell  @%,,8ELL;Q]b]i]i&ellx} 0 1ell 1 1 1(5<<2HTYT`T`rOr7r[namer_cr[U[R5(aL[URSS9 UR b*[R RUR 5 gg[US5(aUR5 gU(a#[US5(aUR5 ggg)zViT weight initialization, original timm impl (for reproducibility). Args: module: Module to initialize. name: Module name for context. needs_reset: If True, call reset_parameters() on modules that have it. rrNrr) rGrjrr"rrdrrrirrr[rr_s rMrrs&"))$$fmm- ;; " GGNN6;; ' #  ( ( );<<!=rOrc[U[R5(aURS5(aT[RR UR 5 [RRURU5 g[RRUR 5 URbXSU;a([RRURSS9O([RR UR5 gg[U[R5(aM[UR 5 URb*[RR UR5 gg[US5(aUR5 gU(a#[US5(aUR5 ggg)zViT weight initialization, matching JAX (Flax) impl. Args: module: Module to initialize. name: Module name for context. head_bias: Bias value for head layer. needs_reset: If True, call reset_parameters() on modules that have it. rsNrprrrr)rGrjr startswithrrrrrdxavier_uniform_rConv2dr#rirr)r[rrr_s rMinit_weights_vit_jaxrs&&"))$$ ??6 " " GGNN6== ) GG  fkk9 5 GG # #FMM 2{{&:?4- 6RWW^^\b\g\gMh' FBII & &fmm$ ;; " GGNN6;; ' #  ( ( );<<!=rOc[U[R5(aSU;a[R"S[ UR RSS-UR RS-5- 5n[RRUR U*U5 O)[RRUR 5 URb*[RRUR5 gg[US5(aUR5 gU(a#[US5(aUR5 ggg) zViT weight initialization, matching moco-v3 impl minus fixed PatchEmbed. Args: module: Module to initialize. name: Module name for context. needs_reset: If True, call reset_parameters() on modules that have it. qkvg@rrr,Nrr)rGrjrrr~rrrruniform_rrdrrirr)r[rr_vals rMinit_weights_vit_mocors&"))$$ D=))Bv}}':':1'='BV]]EXEXYZE['[!\\]C GG  V]]SD# 6 GG # #FMM 2 ;; " GGNN6;; ' #  ( ( );<<!=rOcXU(a#[US5(aUR5 ggg)Nr)rirrs rMinit_weights_reset_parametersr-s%wv'9::!;{rOrcURS5(a[[XS9$URS5(a[[US9$US:Xa[[US9$[[ US9$)NrM)rr_rOr^rL)rrrrrr)rrr_s rMrr2sa u+yZZ  ,+FF 4+NN,+FFrOrfposemb posemb_newgs_new interpolation antialiasc URSU- nURSU- n[[R"U55/S-n[ U5(d#[[R"U55/S-n[ XUUUUSS9$)zRescale the grid of position embeddings when loading from state_dict. *DEPRECATED* This function is being deprecated in favour of using resample_abs_pos_embed r,rT)r"rr r)rrorr~rr%) rrr"rrr ntok_newntok_oldgs_olds rMresize_pos_embedr?s"%66H||A!22H$))H%& '! +F v;;dii)*+a/ !+#  rOmodelrr load_bfloat16c^^^^SSKmT(a SSKJm SSKmSIUUUU4SjjnT(aTRU5nOTRU5nSnSnSnU(dSU;aSnOS U;aS nSnO S U;aS nSn[ UR S 5(GaUR R n [ U S5(+n U (aU O U Rn U RRR[U RRRSU"XRS3555 U RRRU"XRS355 U RRRU"XRS355 U (Gd[U R 5GHup[U R"5GHupUSU S-SUS-S3n[%S5Hn['USUS-35RRU"UUSUS-S355 ['USUS-35RRU"UUSUS-S355 ['USUS-35RRU"UUSUS-S355 M UR(cMUR(RRRU"UUS355 UR(RRRU"UUS355 UR(RRRU"UUS355 GM GM U"XRS 35nOB[UR R*RRSU"XRS 355nURS!SUR R*RRS!S:wa8[-UUR R*RRS!SUUSS"9nUR R*RRU5 UR R*RRU"XRS#355 UR.b%UR.RU"XRS$3SS%95 U(a U"XRS&3SS%9nO U"XRS'3SS%9nURUR0R:waC['US(S5(aSO ['US)S5n[3UUR R4UUUSS*9nUR0RU5 URRRU"XRS+355 URRRU"XRS,355 [7UR8[:R<5(aUS-3U;aUR8RRSXRS-3RS.:Xa`UR8RRU"XRS/355 UR8RRU"XRS-355 UR>GbiUS03nUS1-nUR>R@RU"UUS23SS%95 UR>RBRR[DRF"S3Vs/sH+nU"UUUS3SS%9RIS5RJPM- sn55 UR>RBRR[DRF"S3Vs/sH!nU"UUUS3SS%9RMS.5PM# sn55 UR>RNRRU"UUS43SS%9RIS5RJ5 UR>RNRRU"UUS53SS%9RMS.55 UR>R*RRU"UUS635RIS55 UR>R*RRU"UUS7355 UR>RRRU"UUS8355 UR>RRRU"UUS9355 [%S:5Hn['UR>RPS;US-35RRU"UUSRPS;US-35RRU"UUSunnn[UR"RS55GHupUS?3U;aUS@3nU nO USAU S3nSnUSBUS3-nURTRRU"UUS83USC95 URTRRU"UUS93USC95 URVRXRR[DRF"SDVs/sH,nU"UUUS3SUSE9RIS5RJPM. sn55 URVRXRR[DRF"SDVs/sH"nU"UUUS3SUSE9RMS.5PM$ sn55 URVR*RRU"UUS63USC9RIS55 URVR*RRU"UUS73USC95 URZRRU"UUSFUS3USC95 URZRRU"UUSFUS3USC95 [%S:5Hn['URPS;US-35RRU"UUSGUSHUS3USC95 ['URPS;US-35RRU"UUSGUSHUS3USC95 M GM gs snfs snfs snfs snf)JzULoad weights from .npz checkpoints for official Google Brain Flax implementation rNTcz>UbXnT(aEURTR5RTR5nTR U5nUR S:XaNUR SUR Ss=:XaUR Ss=:XaS:XaO OUR5nU(akUR S:XaUR/SQ5nOGUR S:XaUR/SQ5nO#UR S:XaURSS/5n[R"U5nU$)Nrr,r)rrrr,r)rrr,) rbfloat16astyperarrayndimrflattenrr from_numpy)_wtidxjnpr ml_dtypesnps rM_n2p_load_weights.._n2pas ?B ++,33CKK@B"B 77a$?@A T!hl$;"<=>%hoo6 )%,, 7HA"85QuQUG1=B"1Xa!eW~6==CCDbTQUVWZ[V[U\\cKdIeDfga!eW~6==CCDbTQSTUXYTYSZZ`KaIbDcda!eW~6;;AA$qB4rRSVWRWQXX]I^G_B`a&''3((--44::4RDHXBY@Z;[\((--44::4RD BV@W;XY((--2288a2$l@S>T9UV!87A(89:; '    " " ) ) / / 2DXEU;V9W4XZ "#%"3"3"8"8"?"?"E"Ebc"JJ+     " " ) ) / / 4'   !!'' 5 %%d1x~-F+G&HI " d1xs^#4>?1x}56%@ 1x'OPQUZ[ EOO111!(0@%!H!HAgV[]prsNt, &&00/'   OO+& JJDX-K#L!MNO JJOO$q8+H!IJKL5::ryy))hi A % JJOO ! !! $HI*>(?(E(Eb(I I Q'=%> ?@ d1xy%9#:;<  " , !&EF  $$T!|nE,B*Cu%MN !!'' N^3`N^Dj\!G,- 7 ? ? B D DN^3`)a b %%eiiK[1]K[aDj\!E*+u 5 = =b AK[1]'^ _   &&tA L.I,Je'T'\'\]^'_'a'ab $$T!zl*,E*F%%P%X%XY[%\] ##))$qJAR1S/T*UV !!''Q,?O/P-Q(RSqA EOO''2a!eW 6 = = C CDl^[lmnloovKwIxDy z EOO''2a!eW 6 ; ; A A$qL>YjkljmmrIsGtBu v,6i9GUGell3356X? @A E$X%>?LC$X%>qcCLC!&CG9A$NN    a<.8I(J&KQT!UV tA6F&G$HcRS ##EIIWp/rWpRSDj\!G,-C @ H H K M MWp/r%s t !!%))Tm-oTmqDj\!E*+u# > F Fr JTm-o#p q $$T!zl*,E*FC%P%X%XYZ%[\ ""4ZL*A(B#LM   a<. 7)6(R&SY\!]^ tAj &O$PVYZ[qA EIIAE7| , 3 3 9 9Q,ywqcIJPST V EIIAE7| , 1 1 7 7Q,ywqcGHcR T%73`1]./r-os"2u (u 3u)u state_dictc0n/SQnUR5HupVURU5(dMURUS5nUHnURUSUS5nM US:Xa;SnURSS5n[R "UR S5US'O@US:Xa#S nURS5RS5nOUS :XaURS5nXcU'M U$) N) )conv1patch_embed.proj)positional_embeddingr<)ztransformer.resblocks.rZ)ln_prern)ln_postr)ln_r)in_proj_zqkv.rr)zmlp.c_fcmlp.fc1)z mlp.c_projzmlp.fc2r7rr,r head.weight head.biasclass_embeddingrjr<)itemsrreplacerrzerosr unsqueeze)rMrrout_dictswapsrrsps rM_convert_openai_cliprbs H E  "||F##  IIfb !B "Q%A'A ;A Aq!A$)KK $;H[ ! # #A A((+A +  AA !#" OrOcSSKn0nURSS5 SU;aRURS5US'URS5USSS2S4-US'URS5SS2SS24US'UR5HcupEURSU5(aXSUR S S 5'M2URS U5(aXSUR S S 5'M_XSU'Me U$)Nr mask_tokenregister_tokensrkrjr<r,z(blocks\.(\d+)\.mlp\.w12\.(?:weight|bias)w12fc1z'blocks\.(\d+)\.mlp\.w3\.(?:weight|bias)w3r)repopr[matchr\)rMrrir_rrs rM_convert_dinov2rlsHNN<&J& */@ A *{ ;j>UVWYZVZ>[ [ *{ ;AqrE B  " 88? C C01QYYue, -  XX@! D D/0QYYtU+ ,  # OrOcd0nUR5Hup4URSS5nURSS5nURSS5nURSS5nURS S 5nURS S 5nURS S5nURSS5nXBU'M U$)Nnorm_1rinorm_2rnzpreprocessor.patchifier. patch_embed.zpreprocessor.pos_embedr<ztrunk.r7zpost_trunk_norm.norm.rWz mlp.fc1_gzmlp.fc3z mlp.fc1_x)r[r\)rMrr_rrs rM_convert_aimv2rr0sH  " IIh ( IIh ( II0. A II. < IIh # II(' 2 IIi - IIi - # OrOcSSKnURSU5nSHnURUS5 M /SQn0nUR5HMup6SU;aM UHupxUR XxU5nM US:XaUSSR S5US 'MIXeU'MO 00pUR S 5n UR5Hup6U RU5n U (dXiU'M#U R5upnU RX405nUUU'[U5S :XdM_[R"US US US/SS9U SU SU3'M U $)zH Turn a BEiT-3 checkpoint into a standard VisionTransformer state-dict. rNr)zbeit3.text_embed.weightzbeit3.vision_embed.mask_token))zbeit3\.r7)zvision_embed\.cls_tokenrj)zvision_embed\.rp)zembed_positions\.z pos_embed.)z encoder\.r7)zlayers\.rZ)zffn_layernorm\.rq)zffn\.zmlp.)zself_attn_layer_norm\.znorm1.)z self_attn\.zattn.)zfinal_layer_norm\.znorm2.) inner_attn_lnrrV)z\.A\..z.B.zpos_embed.weightrr<z1blocks\.(\d+)\.attn\.(q|k|v)_proj\.(weight|bias)$rrrrrrZz .attn.qkv.) rirJrjr[subr^compile fullmatchgroups setdefaultrrr)rMrrirrulestmproldnewoutbufpatrrwhichkindstashs rM_convert_beit3rBs_4JJq$J E C  " A: HCs#A " " uq1C F#2 **I JC  MM! F 88:D{B/e u:?3899sU3Zs4!4C'#j/ 0 JrOadapt_layer_scalec SSKn0nURSU5nURSU5nSnSU;a [X5nOSU;a [XSS 9nOS U;a [X5nO[ S UR 555(a [ X5nOS U;aUS nS nOSU;dSU;acSnSU;aZ[UR[R5(a1USUS'[R"USRS5US'OSU;aSnOSU;a [X5nU(aIUR5VV s0sH,upUR!U5(dMU[#U5SU _M. nnn UR5GHiupSU;aUR$R&R(Rupp[#U R5S:a@UR$R&R(RuppU R+U SX5n U RSU :wdU RSU :wa[-U X4UUSS9n OUS:XanU RSUR.RS:waD[1USS 5(aSO [1US!S5n[3U UR$R4UUUSS"9n O*U(aS#U;aUR7S$S%U5nO S&U;aGMeXU'GMl U$s sn nf)'zIconvert patch embedding weight from manual patchify + linear proj to convrNrrMr7zvisual.class_embeddingzmodule.visual.class_embeddingzmodule.visual.)rrdc3,# UH nSU;v M g7f)zbeit3.Nrf).0rs rM 'checkpoint_filter_fn..s 6$5qX]$5sencoderzmodule.zvisual.trunk.pos_embedz"visual.trunk.blocks.0.norm1.weightz visual.trunk.zvisual.head.proj.weightrXrYzmodule.visual.trunk.pos_embedzmodule.visual.trunk.z#preprocessor.patchifier.proj.weightzpatch_embed.proj.weightrrrTr(r<r,r=Fr"r+gamma_z gamma_([0-9])z ls\1.gammar)rirJrbrlanykeysrrGrsrjrrr]rrrr[rrrgrrrr$r<r9r%rrv)rMrrrr rir_rrrOIrrr"s rMcheckpoint_filter_fnr~sH4J j9J F:-)*< (J 6)*DTU  #$Z7 6JOO$5 6 6 6#J6 j  *  !Z /3W[e3e  $ 2z%**bii7X7X&01J&KH] #$)KK ;T0U0[0[\]0^$_H[ ! (J 6' .* <#J6  5?5E5E5G`5GTQ1<%G  ( [hG    rOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Tiny (Vit-Ti/16) r.r1rr4r:rDr<r)vit_tiny_patch16_224rfrrrL model_argsrs rMrr 8s"JJ & s* sX\]gXrkqXr sE LrOc L[SSSSS9n[SSU0[U40UD6D6nU$)z$ViT-Tiny (Vit-Ti/16) @ 384x384. r.rr1rrr)vit_tiny_patch16_384rrs rMrr rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Small (ViT-S/32) rr1rr)vit_small_patch32_224rrs rMrr 8s"JJ & t: tY]^hYslrYs tE LrOc L[SSSSS9n[SSU0[U40UD6D6nU$)z%ViT-Small (ViT-S/32) at 384x384. rrr1rrr)vit_small_patch32_384rrs rMrr rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)ViT-Small (ViT-S/16) r.rr1rrr)vit_small_patch16_224rrs rMrr rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)rr.rr1rrr)vit_small_patch16_384rrs rMrr rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Small (ViT-S/8) rr1rrr)vit_small_patch8_224rrs rMrr s8cqIJ & s* sX\]gXrkqXr sE LrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Base (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k, source https://github.com/google-research/vision_transformer. rr0r1rr)vit_base_patch32_224rrs rMrr 8 s"KJ & s* sX\]gXrkqXr sE LrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Base model (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer. rr0r1rr)vit_base_patch32_384rrs rMrr rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer. r.r0r1rr)vit_base_patch16_224rrs rMrr" rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer. r.r0r1rr)vit_base_patch16_384rrs rMrr, rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Base (ViT-B/8) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer. rr0r1rr)vit_base_patch8_224rrs rMrr6 s8 crJJ & r rW[\fWqjpWq rE LrOc L[SSSSS9n[SSU0[U40UD6D6nU$)znViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929). No pretrained weights. rrr.rr)vit_large_patch32_224rrs rMrr@ 8t2LJ & t: tY]^hYslrYs tE LrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer. rrrr.rr)vit_large_patch32_384rrs rMrrI 8 t2LJ & t: tY]^hYslrYs tE LrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer. r.rrrr)vit_large_patch16_224rrs rMrrS rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer. r.rrrr)vit_large_patch16_384rrs rMrr] rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zViT-Large model (ViT-L/14) rrr.rr)vit_large_patch14_224rrs rMrrg rrOc L[SSSSS9n[SSU0[U40UD6D6nU$)zVViT-Huge model (ViT-H/14) from original paper (https://arxiv.org/abs/2010.11929). rrrr.rr)vit_huge_patch14_224rrs rMrrp s8t2LJ & s* sX\]gXrkqXr sE LrOc N[SSSSSS9n[SSU0[U40UD6D6nU$) zpViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560 rtE]t@(r.r4r:rUrDr<r)vit_giant_patch14_224rrs rMrry s;tuBZ\]J & t: tY]^hYslrYs tE LrOc P[SSSSSS9n[SSU0[U40UD6D6nU$) zpViT-Gigantic (big-G) model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560 r;;@0r.rr)vit_gigantic_patch14_224rrs rMr r  sGtuBZ\]J &" Y/9 Y=A*=WPV=W YE LrOc P[SSSSSS9n[SSU0[U40UD6D6nU$)zViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). Weights taken from: https://github.com/Alibaba-MIIL/ImageNet21K r.r0r1F)r4r:rDr<r=r)vit_base_patch16_224_miilrrs rMr r  sG s"UZ[J &# Z0: Z>B:>XQW>X ZE LrOc X[SSSSSSSSSS9 n[S S U0[U40UD6D6nU$) zAViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 240x240 r.rr1rFr&r r4r:rDr<r;r7r=rXrAr)vit_medium_patch16_gap_240rrs rMrr UB!EtULJ '$ [1; [?CJ?YRX?Y [E LrOc X[SSSSSSSSSS9 n[S S U0[U40UD6D6nU$) zAViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 256x256 r.rr1rFr&rrr)vit_medium_patch16_gap_256rrs rMrr rrOc X[SSSSSSSSSS9 n[S S U0[U40UD6D6nU$) zAViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 384x384 r.rr1rFr&rrr)vit_medium_patch16_gap_384rrs rMrr rrOc X[SSSSSSSSSS9 n[S S U0[U40UD6D6nU$) zBViT-Betwixt (ViT-b/16) w/o class token, w/ avg-pool @ 256x256 r.rr1 Fr&rrr)vit_betwixt_patch16_gap_256rrs rMrr sUB"%EtULJ '% \2< \@DZ@ZSY@Z \E LrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) z?ViT-Base (ViT-B/16) w/o class token, w/ avg-pool @ 224x224 r.r0r1Fr&r4r:rDr<r;r7rAr)vit_base_patch16_gap_224rrs rMrr sPB"%]blqsJ &" Y/9 Y=A*=WPV=W YE LrOc T[SSSSSSSS9n[S SU0[U40UD6D6nU$) z:ViT-Huge model (ViT-H/14) w/ no class token, avg pool rrrr.Fr&rr)vit_huge_patch14_gap_224rrs rMrr PR25^cmrtJ &" Y/9 Y=A*=WPV=W YE LrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) zDViT-Huge model (ViT-H/16) w/ no class token, avg pool @ 448x448 r.rrFr&rr)vit_huge_patch16_gap_448rrs rMr r  rrOc V[SSSSSSSSS9n[S SU0[U40UD6D6nU$) zGViT-Giant (little-gg) model (ViT-g/16) w/ no class token, avg pool r.rrrFr&r4r:rDr<rUr;r7rAr)vit_giant_patch16_gap_224rrs rMr#r# sRR2ue=J '# Z0: Z>B:>XQW>X ZE LrOc h[SSSS[[SS9S9n[S SU0[U40UD6D6nU$) NrrrTrrr:rDr<r?rCr)vit_xsmall_patch16_clip_224rfrrrrs rMr'r' P2TV]^gmqVrsJ &% \2< \@DZ@ZSY@Z \E LrOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) Nrrr1rTrr%r4r:rDr<r?rCr)vit_medium_patch32_clip_224r(rs rMr,r, sVB!dW^_hnrWsuJ &% \2< \@DZ@ZSY@Z \E LrOc h[SSSS[[SS9S9n[S SU0[U40UD6D6nU$) Nrr1rTrr%r&r)vit_medium_patch16_clip_224r(rs rMr.r. r)rOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) Nrrr1rTrr%r+r)vit_betwixt_patch32_clip_224r(rs rMr0r0 sVB"tX_`iosXtvJ && ]3= ]AEjA[TZA[ ]E LrOc j[SSSSS[[SS9S9n[S SU0[U40UD6D6nU$) (ViT-B/32 CLIP image tower @ 224x224 rr0r1Trr%r+r)vit_base_patch32_clip_224r(rs rMr3r3 VB"tX_`iosXtvJ &# Z0: Z>B:>XQW>X ZE LrOc j[SSSSS[[SS9S9n[S SU0[U40UD6D6nU$) z(ViT-B/32 CLIP image tower @ 256x256 rr0r1Trr%r+r)vit_base_patch32_clip_256r(rs rMr6r6% r4rOc j[SSSSS[[SS9S9n[S SU0[U40UD6D6nU$) z(ViT-B/32 CLIP image tower @ 384x384 rr0r1Trr%r+r)vit_base_patch32_clip_384r(rs rMr8r80 r4rOc j[SSSSS[[SS9S9n[S SU0[U40UD6D6nU$) z(ViT-B/32 CLIP image tower @ 448x448 rr0r1Trr%r+r)vit_base_patch32_clip_448r(rs rMr:r:; r4rOc j[SSSSS[[SS9S9n[S SU0[U40UD6D6nU$) zViT-B/16 CLIP image tower r.r0r1Trr%r+r)vit_base_patch16_clip_224r(rs rMr<r<F r4rOc j[SSSSS[[SS9S9n[S SU0[U40UD6D6nU$) z(ViT-B/16 CLIP image tower @ 384x384 r.r0r1Trr%r+r)vit_base_patch16_clip_384r(rs rMr>r>Q r4rOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) z4ViT-Base (ViT-B/16+) CLIP image tower @ 240x240 r.rr1rTrr%r+r)vit_base_patch16_plus_clip_240r(rs rMr@r@\ sVB"tX_`iosXtvJ &( _5? _CG C]V\C] _E LrOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) z0ViT-Large model (ViT-L/14) CLIP image tower rrrr.Trr%r+r)vit_large_patch14_clip_224r(rs rMrBrBg VR2Y`ajptYuwJ &$ [1; [?CJ?YRX?Y [E LrOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) z:ViT-Large model (ViT-L/14) CLIP image tower @ 336x336 rrrr.Trr%r+r)vit_large_patch14_clip_336r(rs rMrErEr rCrOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) z0ViT-Huge model (ViT-H/14) CLIP image tower. rrrr.Trr%r+r)vit_huge_patch14_clip_224r(rs rMrGrG} VR2Y`ajptYuwJ &# Z0: Z>B:>XQW>X ZE LrOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) z9ViT-Huge model (ViT-H/14) CLIP image tower @ 336x336 rrrr.Trr%r+r)vit_huge_patch14_clip_336r(rs rMrJrJ rHrOc j[SSSSS[[SS9S9n[S S U0[U40UD6D6nU$) z9ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378 rrrr.Trr%r+r)vit_huge_patch14_clip_378r(rs rMrLrL rHrOc l[SSSSSS[[SS9S 9n[S S U0[U40UD6D6nU$) zViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560 Pretrained weights from CLIP image tower. rrrrr.Trr%r4r:rUrDr<r?rCr)vit_giant_patch14_clip_224r(rs rMrOrO sX bBY]9$/J '$ [1; [?CJ?YRX?Y [E LrOc l[SSSSSS[[SS9S 9n[S S U0[U40UD6D6nU$) ViT-bigG model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560 Pretrained weights from CLIP image tower. rrr r r.Trr%rNr)vit_gigantic_patch14_clip_224r(rs rMrRrR X bBY]9$/J '' ^4> ^BFzB\U[B\ ^E LrOc l[SSSSSS[[SS9S 9n[S S U0[U40UD6D6nU$) rQrrr r r.Trr%rNr)vit_gigantic_patch14_clip_378r(rs rMrUrU rSrOc l[SSSSS[[SS9SS9n[S S U0[U40UD6D6nU$) r2rr0r1Trr% quick_gelur4r:rDr<r?rCrZr)#vit_base_patch32_clip_quickgelu_224r(rs rMrYrY WB"t9$/<J '- d:D dHLZHb[aHb dE LrOc l[SSSSS[[SS9SS9n[S S U0[U40UD6D6nU$) z/ViT-B/16 CLIP image tower w/ QuickGELU act r.r0r1Trr%rWrXr)#vit_base_patch16_clip_quickgelu_224r(rs rMr\r\ rZrOc l[SSSSS[[SS9SS 9n[S S U0[U40UD6D6nU$) zAViT-Large model (ViT-L/14) CLIP image tower w/ QuickGELU act rrrr.Trr%rWrXr)$vit_large_patch14_clip_quickgelu_224r(rs rMr^r^ WR29$/<J '. e;E eIMjIc\bIc eE LrOc l[SSSSS[[SS9SS 9n[S S U0[U40UD6D6nU$) zKViT-Large model (ViT-L/14) CLIP image tower @ 336x336 w/ QuickGELU act rrrr.Trr%rWrXr)$vit_large_patch14_clip_quickgelu_336r(rs rMrara r_rOc l[SSSSS[[SS9SS 9n[S S U0[U40UD6D6nU$) zAViT-Huge model (ViT-H/14) CLIP image tower w/ QuickGELU act. rrrr.Trr%rWrXr)#vit_huge_patch14_clip_quickgelu_224r(rs rMrcrc WR29$/<J '- d:D dHLZHb[aHb dE LrOc l[SSSSS[[SS9SS 9n[S S U0[U40UD6D6nU$) zJViT-Huge model (ViT-H/14) CLIP image tower @ 378x378 w/ QuickGELU act rrrr.Trr%rWrXr)#vit_huge_patch14_clip_quickgelu_378r(rs rMrfrf rdrOc n[SSSSSS[[SS9S S 9n[S S U0[U40UD6D6nU$) z/ViT-bigG model (ViT-G/14) w/ QuickGELU act rrr r r.Trr%rW)r4r:rUrDr<r?rCrZr)'vit_gigantic_patch14_clip_quickgelu_224r(rs rMrhrhs[bBY]9$/<J '1 h>H hLPQ[Lf_eLf hE LrOc P[SSSSSS9n[SSU0[U40UD6D6nU$) zViT-Base (ViT-B/32+) rrr1rrr4r:rDr<rXr)vit_base_patch32_plus_256rrs rMrkrk%Gs"X\]J &# Z0: Z>B:>XQW>X ZE LrOc P[SSSSSS9n[SSU0[U40UD6D6nU$) zViT-Base (ViT-B/16+) r.rr1rrrjr)vit_base_patch16_plus_240rrs rMrnrn/rlrOc `[SSSSSSS[SS9 n[S SU0[U40UD6D6nU$) z.ViT-Base (ViT-B/16) w/ residual post-norm r.r0r1Frr&) r4r:rDr<r=rXr;rSr7r)vit_base_patch16_rpn_224)rfrrrs rMrprp9sVB"uZ^LeEJ '" Y/9 Y=A*=WPV=W YE LrOc P[SSSSSS9n[SSU0[U40UD6D6nU$) a ViT-Base w/ LayerScale + 36 x 1 (36 block serial) config. Experimental, may remove. Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795 Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow. r.r$rrrjr)vit_small_patch16_36x1_224rrs rMrsrsEsG s"W[\J &$ [1; [?CJ?YRX?Y [E LrOc Z[SSSSS[S9n[SSU0[U40UD6D6nU$) aViT-Small w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove. Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795 Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow. r.rrrr4r:rDr<rXrSr)vit_small_patch16_18x2_224rfrrrs rMrwrwQsM B!XkmJ &$ [1; [?CJ?YRX?Y [E LrOc Z[SSSSS[S9n[SSU0[U40UD6D6nU$) zViT-Base w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove. Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795 r.r0rur1rrvr)vit_base_patch16_18x2_224rxrs rMrzrz^sM B"$YlnJ &# Z0: Z>B:>XQW>X ZE LrOc P[SSSSSS9n[SSU0[U40UD6D6nU$) zFEVA-large model https://arxiv.org/abs/2211.07636 /via MAE MIM pretrainrrrr.r&r4r:rDr<r7r)eva_large_patch14_196rrs rMr}r}jsFt2Y^_J & V,6 V:>z:TV:T VE LrOc N[SSSSSS9n[SSU0[U40UD6D6nU$) zEEVA-large model https://arxiv.org/abs/2211.07636 via MAE MIM pretrainrrrr.r&r|r)eva_large_patch14_336rrs rMrrss;t2Y^_J & t: tY]^hYslrYs tE LrOc N[SSSSSS9n[SSU0[U40UD6D6nU$) zFlexiViT-Small r.rr1rTr4r:rDr<r=r)flexivit_smallrrs rMrr{s;s"Z^_J & mJ mRVWaRlekRl mE LrOc N[SSSSSS9n[SSU0[U40UD6D6nU$)zFlexiViT-Base r.r0r1Trr) flexivit_baserrs rMrrs;s"[_`J & l: lQUV`QkdjQk lE LrOc N[SSSSSS9n[SSU0[U40UD6D6nU$)zFlexiViT-Large r.rrTrr)flexivit_largerrs rMrrs;t2\`aJ & mJ mRVWaRlekRl mE LrOc j[SSSSSS[[SSS9 n[SSU0[U40UD6D6nU$) GViT-Large model (ViT-L/14) w/ parallel blocks and qk norm enabled. r.r0r1TF r4r:rDr<r?r=rCrSr=r>r)vit_base_patch16_xp_224rfrrrrs rMrrsXB"t\`%9ESWJ '! X.8 X<@r=r)vit_small_patch14_reg4_dinov2rrs rMrrsPB!TJ '' ^4> ^BFzB\U[B\ ^E LrOc T[SSSSSSSS9n[S SU0[U40UD6D6nU$) z'ViT-B/14 for DINOv2 w/ 4 registers rr0r1rrTrr)vit_base_patch14_reg4_dinov2rrs rMrrsPB"$TJ '& ]3= ]AEjA[TZA[ ]E LrOc T[SSSSSSSS9n[S S U0[U40UD6D6nU$) z'ViT-L/14 for DINOv2 w/ 4 registers rrrr.rrTrr)vit_large_patch14_reg4_dinov2rrs rMrrsPR24TJ '' ^4> ^BFzB\U[B\ ^E LrOc ~[SSSSSS[[RSSS 9 n[ S S U0[U40UD6D6nU$) rrrrrrrrT) r4r:rDr<rXrUr[rZr>r=r)vit_giant_patch14_reg4_dinov2rrs rMrrs\R24[f"''aPTJ '' ^4> ^BFzB\U[B\ ^E LrOc T[SSSSSSSS9n[S SU0[U40UD6D6nU$) Nrr0r1Fr9 gelu_tanhr4r:rDr<r;r7rZr)vit_base_patch32_siglip_256rrs rMrr$sOB"%]bJ '% \2< \@DZ@ZSY@Z \E LrOc R[SSSSSSS9n[SSU0[U40UD6D6nU$) Nr.r0r1Fr9r4r:rDr<r;r7r)vit_base_patch16_siglip_224rrs rMrr/LB"%]bJ '% \2< \@DZ@ZSY@Z \E LrOc R[SSSSSSS9n[SSU0[U40UD6D6nU$) Nr.r0r1Fr9rr)vit_base_patch16_siglip_256rrs rMrr9rrOc R[SSSSSSS9n[SSU0[U40UD6D6nU$) Nr.r0r1Fr9rr)vit_base_patch16_siglip_384rrs rMrrCrrOc R[SSSSSSS9n[SSU0[U40UD6D6nU$) Nr.r0r1Fr9rr)vit_base_patch16_siglip_512rrs rMrrMrrOc R[SSSSSSS9n[SSU0[U40UD6D6nU$) Nr.rrFr9rr)vit_large_patch16_siglip_256rrs rMrrWLR25^cJ '& ]3= ]AEjA[TZA[ ]E LrOc R[SSSSSSS9n[SSU0[U40UD6D6nU$) Nr.rrFr9rr)vit_large_patch16_siglip_384rrs rMrrarrOc T[SSSSSSSS9n[S SU0[U40UD6D6nU$) Nr.rrFr9rrr)vit_large_patch16_siglip_512rrs rMrrksOR25^cJ '& ]3= ]AEjA[TZA[ ]E LrOc T[SSSSSSSS9n[S S U0[U40UD6D6nU$) Nrr.爅ZӼ @Fr9r4r:rDr<rUr;r7r)vit_so400m_patch14_siglip_224rrs rMrrvOR2]bpuJ '' ^4> ^BFzB\U[B\ ^E LrOc T[SSSSSSSS9n[S S U0[U40UD6D6nU$) Nrrrr.rFr9rr)vit_so400m_patch14_siglip_378rrs rMrrsQR2]bpuJ '' ^4> ^BFzB\U[B\ ^E LrOc T[SSSSSSSS9n[S S U0[U40UD6D6nU$) Nrrrr.rFr9rr)vit_so400m_patch14_siglip_384rrs rMrrrrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) Nr.rrrFr9rr4r:rDr<rUr;r7rZr)vit_so400m_patch16_siglip_256rrs rMrrRR2]bpuJ '' ^4> ^BFzB\U[B\ ^E LrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) Nr.rrrFr9rrr)vit_so400m_patch16_siglip_384rrs rMrrrrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) Nr.rrrFr9rrr)vit_so400m_patch16_siglip_512rrs rMrrrrOc T[SSSSSSSS9n[S SU0[U40UD6D6nU$) Nr.rrFr9rrr)vit_giantopt_patch16_siglip_256rrs rMrrOR25^cJ ') `6@ `DHD^W]D^ `E LrOc T[SSSSSSSS9n[S SU0[U40UD6D6nU$) Nr.rrFr9rrr)vit_giantopt_patch16_siglip_384rrs rMrrrrOc V[SSSSSSSSS9n[S SU0[U40UD6D6nU$) Nrr0r1Fr&rr4r:rDr<r;r7rArZr)vit_base_patch32_siglip_gap_256rrs rMrrsRB"%]blqJ ') `6@ `DHD^W]D^ `E LrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) ]A SigLIP variant of ViT with global average pooling (GAP) instead of attention pooling (MAP).r.r0r1Fr&rr)vit_base_patch16_siglip_gap_224rrs rMrrQB"%]blqJ ') `6@ `DHD^W]D^ `E LrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) rr.r0r1Fr&rr)vit_base_patch16_siglip_gap_256rrs rMrrrrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) rr.r0r1Fr&rr)vit_base_patch16_siglip_gap_384rrs rMrrrrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) rr.r0r1Fr&rr)vit_base_patch16_siglip_gap_512rrs rMrrrrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) rr.rrFr&rr) vit_large_patch16_siglip_gap_256rrs rMrrQR25^cmrJ '* a7A aEI*E_X^E_ aE LrOc T[SSSSSSSS9n[SSU0[U40UD6D6nU$) rr.rrFr&rr) vit_large_patch16_siglip_gap_384rrs rMrrrrOc V[SSSSSSSSS9n[S SU0[U40UD6D6nU$) Nr.rrFr&rrr) vit_large_patch16_siglip_gap_512rrs rMrrsPR255KJ '* a7A aEI*E_X^E_ aE LrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) rrrrr.rFr&r"r)!vit_so400m_patch14_siglip_gap_224rrs rMrr$RR2ueJ '+ b8B bFJ:F`Y_F` bE LrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) rrrrr.rFr&r"r)!vit_so400m_patch14_siglip_gap_378rrs rMrr0rrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) rrrrr.rFr&r"r)!vit_so400m_patch14_siglip_gap_384rrs rMrr<rrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) rrrrr.rFr&r"r)!vit_so400m_patch14_siglip_gap_448rrs rMrrHrrOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) rrrrr.rFr&r"r)!vit_so400m_patch14_siglip_gap_896rrs rMrrTrrOc X[SSSSSSSSSS9 n[S S U0[U40UD6D6nU$) rr.rrrFr&r r4r:rDr<rUr;r7rArZr)!vit_so400m_patch16_siglip_gap_256rrs rMrr`sTR2ue{J '+ b8B bFJ:F`Y_F` bE LrOc X[SSSSSSSSSS9 n[S S U0[U40UD6D6nU$) Nr.rrrFr&rrr)!vit_so400m_patch16_siglip_gap_384rrs rMrrlSR2]b5KJ '+ b8B bFJ:F`Y_F` bE LrOc X[SSSSSSSSSS9 n[S S U0[U40UD6D6nU$) Nr.rrrFr&rrr)!vit_so400m_patch16_siglip_gap_512rrs rMrrwrrOc V[SSSSSSSSS9n[S SU0[U40UD6D6nU$) Nr.rrFr&rrr)#vit_giantopt_patch16_siglip_gap_256rrs rMrrPR255KJ '- d:D dHLZHb[aHb dE LrOc V[SSSSSSSSS9n[S SU0[U40UD6D6nU$) Nr.rrFr&rrr)#vit_giantopt_patch16_siglip_gap_384rrs rMrrrrOc Z[SSSSSSSSS S S 9 n[S S U0[U40UD6D6nU$)Nr.rrrrFTr,r& r4r:rDr<rXrUr;r=r>r7r)vit_wee_patch16_reg1_gap_256rrs rMrrsUB!YZ$1%J '& ]3= ]AEjA[TZA[ ]E LrOc \[SSSSSSSSS S S S 9 n[SS U0[U40UD6D6nU$)Nr.rrrrrFTr,r&r9 r4r:rDr<rXrUr;r=r>r7r:r)vit_dwee_patch16_reg1_gap_256rrs rMrrsXB!YZ$1%\bJ '' ^4> ^BFzB\U[B\ ^E LrOc d[SSSSSSSSSS [S 9 n[S S U0[U40UD6D6nU$) Nr.rrrrFTr,r& r4r:rDr<rXrUr;r=r>r7rSr)vit_pwee_patch16_reg1_gap_256)rfrrrs rMrrsXB!YZ$1%ZnJ '' ^4> ^BFzB\U[B\ ^E LrOc d[SSSSSSSSSS [S 9 n[S S U0[U40UD6D6nU$) Nr.rrrrFTr,r&rr)vit_dpwee_patch16_reg1_gap_256)rfrrrs rMr r sXB!YZ$1%ZrJ '( _5? _CG C]V\C] _E LrOc Z[SSSSSSSSS S S 9 n[S S U0[U40UD6D6nU$)Nr.@rrrffffff@FTr,r&rr)vit_little_patch16_reg1_gap_256rrs rMrrUB!Y\$1%J ') `6@ `DHD^W]D^ `E LrOc \[SSSSSSSSS S S S 9 n[SS U0[U40UD6D6nU$)Nr.r rrrr FTr,r&r9rr) vit_dlittle_patch16_reg1_gap_256rrs rMrrsXB!Y\$1%\bJ '* a7A aEI*E_X^E_ aE LrOc Z[SSSSSSSSS S S 9 n[S S U0[U40UD6D6nU$)Nr.r rrrr FTrr&rr)vit_little_patch16_reg4_gap_256rrs rMrrrrOc X[SSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) Nr.rr1rrFTr,r& r4r:rDr<rXr;r=r>r7r)vit_medium_patch16_reg1_gap_256rrs rMrrRB!$1%J ') `6@ `DHD^W]D^ `E LrOc X[SSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) Nr.rr1rrFTrr&rr)vit_medium_patch16_reg4_gap_256rrs rMrrrrOc X[SSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) Nr.rrrFTrr&rr) vit_mediumd_patch16_reg4_gap_256rrs rMrrRB!$1%J '* a7A aEI*E_X^E_ aE LrOc X[SSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) Nr.rrrrFTrr&rr) vit_mediumd_patch16_reg4_gap_384rrs rMrrrrOc X[SSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) Nr.rr1rrFTr,r&rr) vit_betwixt_patch16_reg1_gap_256rrs rMr!r!RB"$$1%J '* a7A aEI*E_X^E_ aE LrOc X[SSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) Nr.rr1rrFTrr&rr) vit_betwixt_patch16_reg4_gap_256rrs rMr$r$r"rOc X[SSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) Nr.rr1rrFTrr&rr) vit_betwixt_patch16_reg4_gap_384rrs rMr&r&'r"rOc V[SSSSSSSSS9n[S S U0[U40UD6D6nU$) Nr.r0r1FTr&r)r4r:rDr<r;r=r7r>r)vit_base_patch16_reg4_gap_256rrs rMr(r(2sPB"%1J '' ^4> ^BFzB\U[B\ ^E LrOc V[SSSSSSSSS 9n[S S U0[U40UD6D6nU$) ESO150M (shape optimized, but diff than paper def, optimized for GPU) r.rrur~jt@Frr9)r4r:rDr<rUr;r>r7r)vit_so150m_patch16_reg4_map_256rrs rMr,r,=sRB"aUJ ') `6@ `DHD^W]D^ `E LrOc X[SSSSSSSSSS 9 n[S S U0[U40UD6D6nU$) r*r.rrurr+Frr& r4r:rDr<rUr;r>r7rAr)vit_so150m_patch16_reg4_gap_256rrs rMr/r/ITB"aUEJ ') `6@ `DHD^W]D^ `E LrOc X[SSSSSSSSSS 9 n[S S U0[U40UD6D6nU$) r*r.rrurr+Frr&r.r)vit_so150m_patch16_reg4_gap_384rrs rMr2r2Ur0rOc Z[SSSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) HSO150M v2 (shape optimized, but diff than paper def, optimized for GPU) r.@ NN@rFr,r& r4r:rDr<rUrXr=r;r>r7r) vit_so150m2_patch16_reg1_gap_256rrs rMr:r:aWB"[_EaUJ '* a7A aEI*E_X^E_ aE LrOc Z[SSSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) r4r.r5r6r7r8rFr,r&r9r) vit_so150m2_patch16_reg1_gap_384rrs rMr=r=mr;rOc Z[SSSSSSSSSS S 9 n[S S U0[U40UD6D6nU$) r4r.r5r6r7r8rFr,r&r9r) vit_so150m2_patch16_reg1_gap_448rrs rMr?r?yr;rOc T[SSSSSSSS9n[S S U0[U40UD6D6nU$) Nrrrr.rFT)r4r:rDr<rXr@r,r)vit_intern300m_patch14_448rrs rMrArAsNR2EDJ '$ [1; [?CJ?YRX?Y [E LrOc [SSSSSSSSSSS[[S S 9[[S S 9[S 9n[ S S U0[U40UD6D6nU$)ViT Large AIM-v2 model rrrrF@r&silurr%r4r:rDr<r;rArUr7r=r@rZrCrRr[r)aimv2_large_patch14_224rfrrrrrs rMrGrGpR1%Y^EEUV\7-UY@ZflJ '! X.8 X<@r7rXr,r) test_vit2rrs rMrnrnBsK1QaU_ceJ ' hz hTR\Mg`fMg hE LrOc X[SSSSSSSSS S S 9 n[S S U0[U40UD6D6nU$)rjr.` rrFr,r9Tr) r4r:rDr<rUr;r>r7rBrXr) test_vit3rrs rMrrrrMsL1QaUPTbfhJ ' hz hTR\Mg`fMg hE LrOc Z[SSSSSSSSSS S S 9 n[S S U0[U40UD6D6nU$)rjr.rprqrFr,r&rTrmsnorm) r4r:rDr<rUr;r>r7rXr,rCr) test_vit4rrs rMruruXsM1QaU_cJ ' hz hTR\Mg`fMg hE LrOc p[SSSSSSSSS[[SS9S 9 n[S S U0[U40UD6D6nU$) zlBEiT3 Base model (ViT-Base size) with patch size 16x16. Remapped to VisionTransformer with scale_norm=True. r.r0r1rTr&rr% r4r:rDr<rUrVrWr;r7rCr)beit3_base_patch16_224r(rs rMrxrxesS B"TtQV9$/J ' uJ uZ^_iZtmsZt uE LrOc p[SSSSSSSSS[[SS9S 9 n[S S U0[U40UD6D6nU$) znBEiT3 Large model (ViT-Large size) with patch size 16x16. Remapped to VisionTransformer with scale_norm=True. r.rrrTr&rr%rwr)beit3_large_patch16_224r(rs rMrzrzssS R2TtQV9$/J ' vZ v[_`j[unt[u vE LrOc p[SSSSSSSSS[[SS 9S 9 n[S S U0[U40UD6D6nU$) z]BEiT3 Giant model with patch size 14x14. Remapped to VisionTransformer with scale_norm=True. rrrr.8mt@Tr&rr%rwr)beit3_giant_patch14_224r(rs rMr}r}sS R2TtQV9$/J ' vZ v[_`j[unt[u vE LrOc r[SSSSSSSSSS[[S S 9S 9 n[S S U0[U40UD6D6nU$)ztBEiT3 Giant model with patch size 14x14 and image size 336x336. Remapped to VisionTransformer with scale_norm=True. rrrrr.r|Tr&rr%) r3r4r:rDr<rUrVrWr;r7rCr)beit3_giant_patch14_336r(rs rMrrsV t2W]TtQV9$/J ' vZ v[_`j[unt[u vE LrOvit_tiny_patch16_224_in21kvit_small_patch32_224_in21kvit_small_patch16_224_in21kvit_base_patch32_224_in21kvit_base_patch16_224_in21kvit_base_patch8_224_in21kvit_large_patch32_224_in21kvit_large_patch16_224_in21kvit_huge_patch14_224_in21kvit_base_patch32_224_samzvit_base_patch32_224.samvit_base_patch16_224_samzvit_base_patch16_224.samvit_small_patch16_224_dinovit_small_patch8_224_dinovit_base_patch16_224_dinovit_base_patch8_224_dinovit_base_patch16_224_miil_in21k!vit_base_patch32_224_clip_laion2b)"vit_large_patch14_224_clip_laion2b!vit_huge_patch14_224_clip_laion2b"vit_giant_patch14_224_clip_laion2b)FFFTrTrTNr)r%r,Fr)r7rTT)rMrTT)r,rfrF)r7F)zvisual.)FrTr)FNr(rcopyloggingros collectionsr functoolsrtypingrrrrr r r r r r ImportErrortyping_extensionsrtorch.nnrjtorch.nn.functional functionalr torch.jitr timm.datarrrrrr timm.layersrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+_builderr- _featuresr. _manipulater/r0r1r2 _registryr3r4r5__all__ getLoggerr_loggerrIrorrrrNrQrrrrrrHr*r6rrrrrrr|rrbrlrrrfrrr default_cfgsr[rJ_quick_gelu_cfgsrdeepcopycr\environlowerrrrrrrrrrrrrrrrrrrrrrr r rrrrrrr r#r'r,r.r0r3r6r8r:r<r>r@rBrErGrJrLrOrRrUrYr\r^rarcrfrhrkrnrprsrwrzr}rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrrrrrrr!r$r&r(r,r/r2r:r=r?rArGrMrQrVrYr[r^r`rbrdrfrhrlrnrrrurxrzr}r)rrs00rMrs@2  #OOO*  0++RRYY     H %     04!! !! !  !  !!!!T"))_-!!YY!HQBIIQhC299CLy299yxeryyePQ"))Ql!!"&+  <<    $  2x xv""))"3"$"Z^"( " """ "  "@""))"3"$"Z^".""))"3"RV"bf" Gs Gu GQU Gai G "#"$&  LLc3h      \\2WT*WTSWT#WTcgWTtxWTWTz #ell*+# ## #u||  #Lell*+  #u|| .ell*+  #u|| $9t9~#(& Lell*+L L L L  L  #u||  L^cT#s(^"|1$3| 02 |0 d2|0 d]S2B|"1$ e3#|*1$ e]S3B+|21$ e33|:1$ e]S3B;|B0 f2C|J0 d]S2BK|R0 e2S|Z0 e]S2B[|b/ d1c|j1$ e3k|r1$ e]S3Bs|~.t }0|D.t } 300E|L/ ~ 310M|X( d*Y|`( d]S*Ba|h' d)i|p' d]S)Bq|x' d)y|@' d]S)BA|J&t|K|L%drlM|N&t|O|P)$2,Q|V&t(W|^&t(_|f')g|n')o|v&t(w|@( {e*-A|H)$ |e+-I|P)$ |e+-Q|X( }e*-Y|`( |e*-a|h' {e)-i|p)$ |e+-q||$T HVZ&}|B$T HVZ&C|L!$ i "(t 3@0q!|v!6t 380w!|~!>t 3@0!|D">t 3(@DE"|L"+D "(< 3A -M"|X"+D "(< 3A-Y"|d"' ?L!)%e"|l",T ?L!.%m"|t"&t ?L!(%u"||"$T ?L!&%}"|D#$T ?L!&%E#|L#' ?L 3A)?M#|T#,T ?L 3A.?U#|\#&t ?L 3A(?]#|d#$T ?L 3A&?e#|l#$T ?L 3A&?m#|t#' ?L 3A)?u#||#&t ?L 3A(?}#|D$$T ?L 3A&?E$|L$$T ?L 3A&?M$|V$$ 41W$|\$  41  41  41-1 "(' !<' !<' :' &'H!'"()L#'$+O)L*N)'uS*))*r[os.Ax$FAx7F% Ax7F5Ax7x$ Ax4x3Ax4