RЦi6SrSSKrSSKJr SSKJrJrJr SSKJ r SSK J r J r J r JrJrJrJrJr SSKrSSKJr SSKJr SS KJrJr SS KJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3 S S K4J5r5 S S K6J7r7 S SK8J9r9 S SK:J;r;Jr>J?r? /SQr@\"SS55rA\"SS55rB\"SS55rC"SS\R5rE"SS\R5rF"SS\R5rGGSS\RS \HS!\HS"S4S#jjrI"S$S%\R5rJGSS\RS \HS!\HS"S4S&jjrKS'\\LS(\LS"\L4S)jrM"S*S+\R5rN"S,S-\R5rOS.\RS/\\LS"\R4S0jrQ\9S1\RS/\\LS2\\LS"\R4S3j5rRS.\RS4\\LS"\R4S5jrS\9S1\RS4\\LS2\\LS"\R4S6j5rTS7\AS/\\L\L4S"\\ 4S8jrU"S9S:\R5rV"S;S<\R5rWS.\RS/\\LS"\R4S=jrX\9S1\RS/\\LS2\\LS"\R4S>j5rYS.\RS4\\LS"\R4S?jrZ\9S1\RS4\\LS2\\LS"\R4S@j5r["SASB\R5r\"SCSD\R5r]"SESF\R5r^"SGSH\R5r_"SISJ\R5r`S7\AS2\\L\L4S"\A4SKjraS7\CSL\ S"\C4SMjrb"SNSO\R5rcGSSU\HSV\HSW\dSX\dSY\HSZ\HS[\dS\\HS]\HS^\\eS_\HS`\LS"\ \H\ 44SajjrfGSSU\HSV\HSW\dSc\eSZ\HS\\HS]\HS/\\\L\L4Sd\LS^\\eS_\HS`\LS"\ \H\ 44SejjrgGSSU\HSV\HSZ\HSh\HS\\HS]\HS/\\\L\L4Si\dS^\\e\\e\e44S_\HS`\LS"\ \H\ 44SjjjrhS"\ \H\ 44Skjri\j"GS0Sl\C"GSSmSnSoSp.\g"SRSqSr9D6_Ss\C"GSSmStSoSp.\g"SPSRSqSu9D6_Sv\C"GSSwSxSoSp.\f"SRSQSy9D6_Sz\C"GSSwS{SoSp.\f"SbSRSQS|9D6_S}\C"GSS~S{SSp.\f"SbSS9D6_S\C"GSSS{SSp.\f"SbSSfS9D6_S\C"GSSwSxSoSp.\f"SbSRSQSS9D6_S\C"GSSmStSoSp.\g"SRSqSgSS9D6_S\C"GSSwSxSoSp.\f"SbSgS9D6_S\C"GSSwS{SoSp.\f"SSRSQSgSS9D6_S\C"GSSwS{SoSp.\f"SbSgSTS9D6_S\C"GSS~S{SSp.\f"SbSSfSgS9D6_S\C"GSSS{SSp.\f"SbSSfSgS9D6_S\C"GSSmStSoSS.\f"5D6_S\C"GSSmStSoSS.\h"SSSS9D6_S\C"SwSnSSS9_S\C"SwS{SSS9_S\C"S~S{SSS9_S\C"SS{SSS9_S\C"SSSSS9_S\C"SSSSS9_S\C"GSSSSSS.\g"5D6_S\C"GSSmSSSoS.\g"5D6_S\C"GSSmSSSoS.\g"5D6_S\C"GSSmSSSoS.\g"5D6_S\C"GSSSSSS.\g"SgS9D6_S\C"GSSmSSSoS.\g"SgS9D6_S\C"GSSmSSSoS.\g"SgS9D6_S\C"GSSwSSSoS.\g"SgSfS9D6_S\C"GSSwS{SSoSS.\g"SgS9D6_S\C"GSSmSSSoSS.\h"5D6_S\C"GSSmSSSoS.\h"5D6_S\C"GSSwSSSS.\h"5D6_S\C"GSSwSSSSS.\h"SRSSS9D6_S\C"GSS~SSSS.\h"SRS9D6_S\C"GSSSSSSS.\h"SRS9D6_S\C"GSSmSSSSRSTS.\i"5D6_S\C"GSSwSSSSRSS.\i"5D6_S\C"GSSwS{SSSRSS.\i"5D6_S\C"GSS~S{SSSRSS.\i"5D6_S\C"GSSS{SSSRSS.\i"5D6_6rkS\ \H\R4S\RS"\ \H\R44SjrlGSS\HS\\HS\dSL\ S"\c4 SjjrmGSS\HSL\ S"\ \H\ 44Sjjrn\>"0S\n"SS9_S\n"SSSS9_S\n"SSS9_S\n"SSS9_S\n"SS9_S\n"SS9_S\n"SS9_S\n"SSSSSS9_S\n"SS\\SS9_S\n"SSSS9_S\n"SS9_S\n"SSS9_S\n"SSS9_S\n"SS9_GS\n"SS9_GS\n"SGSSS9_GS\n"SGSGS9_0GS\n"SGSGS9_GS\n"SGSGS9_GS\n"SGSGS9_GS \n"SS9_GS \n"SS9_GS \n"SS9_GS \n"SS9_GS \n"SS9_GS\n"SS9_GS\n"SGSGSGS9_GS\n"SGSGSGSGS9_GS\n"SGSS9_GS\n"SGSGSGS9_GS\n"SGSGSGS9_GS\n"SGSGSGSGS9_GS\n"SGSGSGSGS9_GS\n"SGSGSGSGS9_E0GS \n"SGS!SS9_GS"\n"SGSGSGS9_GS#\n"SS9_GS$\n"SSSSSS9_GS%\n"SGSGS9_GS&\n"SGS'GSGSGS9_GS(\n"SGSGSGS9_GS)\n"SGS*GSGSGS9_GS+\n"SGSGSGS,9_GS-\n"SS9_GS.\n"SSSSSS9_GS/\n"SS9_GS0\n"SGSGS9_GS1\n"S\\GS29_GS3\n"SSSSSS9_GS4\n"SGS5GS6SSS9_GS7\n"S\\GS29_E0GS8\n"SSSSSS9_GS9\n"SGS5GS6SSS9_GS:\n"S\\GS29_GS;\n"SSSSSS9_GS<\n"SGS5GS6SSS9_GS=\n"S\\GS29_GS>\n"SSSSSS9_GS?\n"SGS5GS6SSS9_GS@\n"SGSAGS9_GSB\n"SSSSSS9_GSC\n"SGS5GS6SSS9_GSD\n"SGSAGS9_GSE\n"SSSSSS9_GSF\n"SGS5SSGSG9_GSH\n"SGSAGS9_GSI\n"SSSSSS9_GSJ\n"SGS5GS6SSS9_E5ro\?GSS\dSL\ S"\c4GSKjj5rp\?GSS\dSL\ S"\c4GSLjj5rq\?GSS\dSL\ S"\c4GSMjj5rr\?GSS\dSL\ S"\c4GSNjj5rs\?GSS\dSL\ S"\c4GSOjj5rt\?GSS\dSL\ S"\c4GSPjj5ru\?GSS\dSL\ S"\c4GSQjj5rv\?GSS\dSL\ S"\c4GSRjj5rw\?GSS\dSL\ S"\c4GSSjj5rx\?GSS\dSL\ S"\c4GSTjj5ry\?GSS\dSL\ S"\c4GSUjj5rz\?GSS\dSL\ S"\c4GSVjj5r{\?GSS\dSL\ S"\c4GSWjj5r|\?GSS\dSL\ S"\c4GSXjj5r}\?GSS\dSL\ S"\c4GSYjj5r~\?GSS\dSL\ S"\c4GSZjj5r\?GSS\dSL\ S"\c4GS[jj5r\?GSS\dSL\ S"\c4GS\jj5r\?GSS\dSL\ S"\c4GS]jj5r\?GSS\dSL\ S"\c4GS^jj5r\?GSS\dSL\ S"\c4GS_jj5r\?GSS\dSL\ S"\c4GS`jj5r\?GSS\dSL\ S"\c4GSajj5r\?GSS\dSL\ S"\c4GSbjj5r\?GSS\dSL\ S"\c4GScjj5r\?GSS\dSL\ S"\c4GSdjj5r\?GSS\dSL\ S"\c4GSejj5r\?GSS\dSL\ S"\c4GSfjj5r\?GSS\dSL\ S"\c4GSgjj5r\?GSS\dSL\ S"\c4GShjj5r\?GSS\dSL\ S"\c4GSijj5r\?GSS\dSL\ S"\c4GSjjj5r\?GSS\dSL\ S"\c4GSkjj5r\?GSS\dSL\ S"\c4GSljj5r\?GSS\dSL\ S"\c4GSmjj5r\?GSS\dSL\ S"\c4GSnjj5r\?GSS\dSL\ S"\c4GSojj5r\?GSS\dSL\ S"\c4GSpjj5r\?GSS\dSL\ S"\c4GSqjj5r\?GSS\dSL\ S"\c4GSrjj5r\?GSS\dSL\ S"\c4GSsjj5r\?GSS\dSL\ S"\c4GStjj5r\?GSS\dSL\ S"\c4GSujj5r\?GSS\dSL\ S"\c4GSvjj5r\?GSS\dSL\ S"\c4GSwjj5r\?GSS\dSL\ S"\c4GSxjj5r\?GSS\dSL\ S"\c4GSyjj5r\?GSS\dSL\ S"\c4GSzjj5r\?GSS\dSL\ S"\c4GS{jj5r\?GSS\dSL\ S"\c4GS|jj5r\?GSS\dSL\ S"\c4GS}jj5r\?GSS\dSL\ S"\c4GS~jj5r\?GSS\dSL\ S"\c4GSjj5r\?GSS\dSL\ S"\c4GSjj5r\?GSS\dSL\ S"\c4GSjj5r\?GSS\dSL\ S"\c4GSjj5rg(aMaxVit and CoAtNet Vision Transformer - CNN Hybrids in PyTorch This is a from-scratch implementation of both CoAtNet and MaxVit in PyTorch. 99% of the implementation was done from papers, however last minute some adjustments were made based on the (as yet unfinished?) public code release https://github.com/google-research/maxvit There are multiple sets of models defined for both architectures. Typically, names with a `_rw` suffix are my own original configs prior to referencing https://github.com/google-research/maxvit. These configs work well and appear to be a bit faster / lower resource than the paper. The models without extra prefix / suffix' (coatnet_0_224, maxvit_tiny_224, etc), are intended to match paper, BUT, without any official pretrained weights it's difficult to confirm a 100% match. Papers: MaxViT: Multi-Axis Vision Transformer - https://arxiv.org/abs/2204.01697 @article{tu2022maxvit, title={MaxViT: Multi-Axis Vision Transformer}, author={Tu, Zhengzhong and Talebi, Hossein and Zhang, Han and Yang, Feng and Milanfar, Peyman and Bovik, Alan and Li, Yinxiao}, journal={ECCV}, year={2022}, } CoAtNet: Marrying Convolution and Attention for All Data Sizes - https://arxiv.org/abs/2106.04803 @article{DBLP:journals/corr/abs-2106-04803, author = {Zihang Dai and Hanxiao Liu and Quoc V. Le and Mingxing Tan}, title = {CoAtNet: Marrying Convolution and Attention for All Data Sizes}, journal = {CoRR}, volume = {abs/2106.04803}, year = {2021} } Hacked together by / Copyright 2022, Ross Wightman N) OrderedDict) dataclassreplacefield)partial)AnyCallableDictListOptionalSetTupleUnion)nn)Final)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)MlpConvMlpDropPathcalculate_drop_path_rates LayerNorm LayerScale LayerScale2dClassifierHeadNormMlpClassifierHead create_attn get_act_layerget_norm_layerget_norm_act_layer create_conv2d create_pool2dtrunc_normal_tf_ to_2tuple extend_tuplemake_divisible_assert RelPosMlp RelPosBias RelPosBiasTfuse_fused_attnresize_rel_pos_bias_table)build_model_with_cfg)feature_take_indices)register_notrace_function) named_applycheckpoint_seq)generate_default_cfgsregister_model) MaxxVitCfgMaxxVitConvCfgMaxxVitTransformerCfgMaxxVitcl\rSrSr%SrSr\\S'Sr\ \S'Sr \ \S'Sr \ \S 'Sr \ \S 'Sr\ \S 'S r\ \S 'S r\ \S'Sr\\S'Sr\\S'Sr\\S'Sr\\S'Sr\\\\4\S'Sr\\\\4\S'Sr\ \S'Sr\ \S'Sr\\ \S'Sr\\S'Sr\\S 'S!r\\S"'S#r \ \S$'S%r!S&r"g)'r7Tz-Configuration for MaxxVit transformer blocks. dim_headT head_first@ expand_ratio expand_first shortcut_bias attn_bias attn_drop proj_dropavg2 pool_typebias rel_pos_type rel_pos_dimpartition_ratioN window_size grid_sizeF no_block_attn use_nchw_attn init_valuesgelu act_layer layernorm2d norm_layer layernorm norm_layer_clư>norm_epscURb[UR5UlURb:[UR5UlURcURUlgggN)rNr$rMselfs R/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/maxxvit.py __post_init__#MaxxVitTransformerCfg.__post_init__ms\ >> %&t~~6DN    '()9)9:D ~~%!%!1!1& ()rNrM)#__name__ __module__ __qualname____firstlineno____doc__r<int__annotations__r=boolr?floatr@rArBrDrErGstrrIrKrLrMr rrNrOrPrQrSrUrWrYr___static_attributes__rar^r7r7Ts7HcJL%L$M4ItIuIuIsL#KOS-1K%S/*1+/Ixc3h(/M4M4#'K%'Is#J#$M3$He2rar7cD\rSrSr%SrSr\\S'Sr\ \S'Sr \ \S'S r \ \S 'S r\ \S 'S r\ \S'Sr\ \S'Sr\\S'Sr\\S'Sr\\S'Sr\\S'S r\ \S'Sr\\S'Sr\\S'Sr\ \S'Sr\\ \S'S r\\S!'Sr\\S"'Sr\\S#'S$r\\ \S%'S&rS'r g$)(r6vz-Configuration for MaxxVit convolution blocks.mbconv block_typer>r?T expand_output kernel_sizer- group_sizeF pre_norm_act output_biasdw stride_moderFrGdownsample_pool_typepadding attn_earlyse attn_layersiluattn_act_layer? attn_ratiorXrQrRrSrUrWNrYcHURS;deURS:HnUR(dU(aSOSUlUR(dU(dSUlURcU(aSOSUlUR=(d UR Ulg)N)rpconvnextrp batchnorm2drTrVh㈵>rX)rqrUrWrYrzrG)r] use_mbconvs r^r_MaxxVitConvCfg.__post_init__st"8888__0 /9m}DO!!*!,D  == $.DDDM$($=$=$O!ra)rzrYrUrW)!rbrcrdrerfrqrkrhr?rjrrrirtrgrurvrwryrGrzr|r}rrrrQr rSrUrWrYr_rlrmrar^r6r6vs7JL%M4KJL$KKIs &#&GSJJ NC J#'K%'IsJM3 $Hhuo$ Prar6c\rSrSr%SrSr\\S4\S'Sr \\S4\S'Sr \\ \ \\ S44S4\S 'S r \ \\\\44\S 'S r\\S '\"\S9r\\S'\"\S9r\\S'Sr\\\S'Sr\ \S'Srg)r5z!Configuration for MaxxVit models.`. embed_dimrsrdepths)CrTrrq@ stem_widthF stem_bias)default_factoryconv_cfgtransformer_cfgNhead_hidden_sizevit_eff weight_initrm)rbrcrdrerfrrrgrhrrqrrkrrrirr6rr7rrr rrlrmrar^r5r5s+!4IuS#X4*FE#s(O*:NJeCsCx01367N.0Jc5c?*+0It$^DHnD-2CX-YO*Y&*hsm* K rar5c^\rSrSr%Sr\\\S'SS\S\ \S\S\S\S \S \ \ S \ S \ 4U4S jjjr SS\ RS\ \ RS\ R4SjjrSrU=r$) Attention2dz)Multi-head attention for 2D NCHW tensors. fused_attndimdim_outr<rHr@r= rel_pos_clsrDrEc >XS.n [TU]5 U=(d UnU(aUOUn X-UlX0lX`lUS-Ul[ 5Ul[R"XS-S4SU0U D6Ul U(aU"SSUR0U D6OSUl [R"U5Ul [R"XS4SU0U D6Ul[R"U 5Ulg)  Args: dim: Input dimension. dim_out: Output dimension (defaults to input dimension). dim_head: Dimension per attention head. bias: Whether to use bias in qkv and projection. expand_first: Whether to expand channels before or after qkv. head_first: Whether heads are first in tensor layout. rel_pos_cls: Relative position class to use. attn_drop: Attention dropout rate. proj_drop: Projection dropout rate. devicedtypersr-rH num_headsNrm)super__init__rr<r=scaler+rrConv2dqkvrel_posDropoutrDprojrEr]rrr<rHr@r=rrDrErrdddim_attn __class__s r^rAttention2d.__init__s4 / .S*7!- $% (*99SQ,CCCFQ{BT^^BrBW[ I.IIhDDD I.raxshared_rel_posreturncURup4pVUR(aKURU5RX0RUR S-S5R SSS9upxn OJURU5RUSURUR S5RS5upxn UR(aSn URbURR5n OUbUn [RRRUR!SS5R#5UR!SS5R#5U R!SS5R#5U UR$(aUR&R(OSS9R!SS5RUSXV5nOXpR*-nUR!SS5U-n URbURU 5n OUbX-n U R-SS9n UR'U 5n XR!SS5-RUSXV5nUR/U5nUR1U5nU$) Nrsrrr-rC attn_mask dropout_p)shaper=rviewrr<chunkreshapeunbindrrget_biastorchr functionalscaled_dot_product_attention transpose contiguoustrainingrDprsoftmaxrrE) r]rrBrHWqkvrBattns r^forwardAttention2d.forwardsWW a ??hhqk&&q..$--!:KRPVVWX^_V`GA!hhqk))!Q rRYYZ[\GA! ??I||' LL113 +* ##@@ B#..0 B#..0 B#..0#.2mm$..** A iB2q 4 JJA;;r2&*D||'||D)+,<>$'D^^B++11!R>A IIaL NN1 ra rDr<rr=rrrErrr Nr;TTTNrCrCNNr[rbrcrdrerfrrirhrgr r rjrrTensorrrl __classcell__rs@r^rrs3d  &*!%#.2!!(/(/c](/ (/  (/  (/(/"(+(/(/(/(/T##x 7M#Y^YeYe##rarc^\rSrSr%Sr\\\S'SS\S\ \S\S\S\S \S \ \ S \ S \ 4U4S jjjr SS\ RS\ \ RS\ R4SjjrSrU=r$) AttentionClz/Channels-last multi-head attention (B, ..., C).rrrr<rHr@r=rrDrEc >XS.n [TU]5 U=(d UnU(aX!:aUOUn X-S:XdS5eX-UlX0lX`lUS-Ul[ 5Ul[R"XS-4SU0U D6Ul U(aU"S SUR0U D6OSUl [R"U5Ul [R"X4SU0U D6Ul[R"U 5Ulg) rrrz(attn dim should be divisible by head_dimrrsrHrNrm)rrrr<r=rr+rrLinearrrrrDrrErs r^rAttentionCl.__init__s4 / .S*w}7#"a'S)SS'!- $% (*99SQ,@T@R@FQ{BT^^BrBW[ I.IIhAdAbA I.rarrrcTURSnURSSnUR(a\URU5RUSURUR S-5R SS5RSSS9upVnOZURU5RUSSURUR 5R SS5RS5upVnUR(aSnURbURR5nOUbUn[RRR!XVUUUR"(aUR$R&OSS9nOjXPR(-nXVR S S5-n URbURXS 9n OUbX-n U R+SS9n UR%U 5n X-nUR SS5RUS -5nUR-U5nUR/U5nU$) Nrrrsr-rrrCrrr)r)rr=rrrr<rrrrrrrrrrrrrDrrrrrE) r]rrr restore_shaperrrrBrs r^rAttentionCl.forward,s GGAJ ??hhqk&&q"dnndmma>OPZZ[\^_`ffghnofpGA!hhqk))!RDNNDMMR\\]^`abiijklGA! ??I||' LL113 +* ##@@a#.2mm$..**AA JJA{{2r**D||'||D|H+,<>$'DA KK1  % %me&; < IIaL NN1 rarrr[rrs@r^rrs9d  &*!%#.2!!)/)/c])/ )/  )/  )/)/"(+)/)/)/)/V##x 7M#Y^YeYe##rarc ^\rSrSrSrS S\S\S\S\S\4 U4SjjjrS \ RS \ R4S jr S r U=r $) Downsample2diRa!A downsample pooling module supporting several maxpool and avgpool modes. * 'max' - MaxPool2d w/ kernel_size 3, stride 2, padding 1 * 'max2' - MaxPool2d w/ kernel_size = stride = 2 * 'avg' - AvgPool2d w/ kernel_size 3, stride 2, padding 1 * 'avg2' - AvgPool2d w/ kernel_size = stride = 2 rrrGr|rHc >[TU]5 US;deUS:Xa[SSSU=(d SS9UlO[US:Xa[SSU=(d SS 9UlO;US :Xa[S SSS U=(d SS 9UlO[S SU=(d SS 9UlX:wa[R "XSXVUS 9Ulg[R"5Ulg)z Args: dim: Input dimension. dim_out: Output dimension. pool_type: Type of pooling operation. padding: Padding mode. bias: Whether to use bias in expansion conv. )maxmax2avgrFrrsrr-)rtstrider|rr)r|rF)rtrcount_include_padr|)rHrrN)rrr"poolrrexpandIdentity) r]rrrGr|rHrrrs r^rDownsample2d.__init__[s$ ::::  %e1glYZ[DI & %eQ 1EDI % %1Q%QXQ]\]_DI&eQ 1EDI >))C!$UZ[DK++-DKrarrcJURU5nURU5nU$r[)rrr]rs r^rDownsample2d.forward~s! IIaL KKNra)rr)rFr{TNN)rbrcrdrerfrgrkrirrrrrlrrs@r^rrRst$!(!(!( !(  !(  !(!(F%,,rarr{modulenameschemerc[U[R[R45(GaUS:Xa`[RR UR SS9 URb*[RRUR5 ggUS:XaL[UR SS9 URb*[RRUR5 ggUS:Xaa[RRUR 5 URb*[RRUR5 gg[RRUR 5 URbYSU;a)[RR URSS9 g[RRUR5 ggg) z&Initialize transformer module weights.normal{Gz?stdN trunc_normal xavier_normalmlprX) isinstancerrrinitnormal_weightrHzeros_r#xavier_normal_xavier_uniform_)rrrs r^_init_transformerrs8&299bii011 X  GGOOFMMsO 3{{&v{{+' ~ % V]] 4{{&v{{+'  & GG " "6== 1{{&v{{+' GG # #FMM 2{{&D=GGOOFKKTO:GGNN6;;/ '!2rac ^\rSrSrSrSS\"5SSS4S\S\S\S \\S \S \ 4 U4S jjjr SS \ SS4Sjjr SS\ RS\\ RS\ R4SjjrSrU=r$)TransformerBlock2diaETransformer block with 2D downsampling. '2D' NCHW tensor layout Some gains can be seen on GPU using a 1D / CL block, BUT w/ the need to switch back/forth to NCHW for spatial pooling, the benefit is minimal so ended up using just this variant for CoAt configs. This impl was faster on TPU w/ PT XLA than the 1D experiment. r-NrCrrrrcfg drop_pathc >XxS.n [T U]5 [[UR5UR S9n [ UR5n US:Xap[X4URURS.U D6Ul [R"[SU "U40U D64S[X4SUR0U D64/55UlO/X:Xde[R "5Ul U "U40U D6Ul[#UU4UR$UR&UR(UUR*UR,S.U D6UlUR0(a[3U4S UR00U D6O[R "5UlUS :a [7U5O[R "5UlU "U40U D6Ul[=S U[?X%R@-5U UR,S .U D6Ul!UR0(a[3U4S UR00U D6O[R "5Ul"US :a[7U5Ul#g [R "5Ul#g )z Args: dim: Input dimension. dim_out: Output dimension. stride: Stride for downsampling. rel_pos_cls: Relative position class. cfg: Transformer block configuration. drop_path: Drop path rate. repsr)rGrHnormdownrG)r<r@rHrrDrErQrC in_featureshidden_featuresrSdropNrm)$rrrrrUrYrrSrrGrAshortcutr Sequentialrnorm1rrr<r@rBrDrErrQrls1r drop_path1norm2rrgr?rls2 drop_path2) r]rrrrrrrrrrUrSrs r^rTransformerBlock2d.__init__s ( / ^CNN;N !#--0 Q;(mUXUfUfmjlmDM{C.2./cN#--N2NO4(DJ > !>KKMDM#C.2.DJ    \\))#mmmm     PS<KS__KKdfdododq1:R(9-R[[].2.  *:*: :;    PS<KS__KKdfdododq1:R(9-R[[]rarrc4[[[US9U5 gNr)r1rrr]rs r^ init_weightsTransformerBlock2d.init_weightssG-f=tDrarrc $URU5URURURUR U5US955-nXR UR URURU5555-nU$)Nr) rr rrrr#r"rr!)r]rrs r^rTransformerBlock2d.forwardsm MM! ttxx $**Q-`n 8o/pq q $**Q-)@ AB Bra) rr r#rr"rrr!rr{r[)rbrcrdrerfr7rgr r rjrrkr)rrrrlrrs@r^rrs.2)>)@!;S;S;S ;S "(+ ;S ' ;S;S;SzE3EEx 7MY^YeYerarc[U[R5(GaUS:Xa`[RR UR SS9 UR b*[RRUR 5 ggUS:XaL[UR SS9 UR b*[RRUR 5 ggUS:Xaa[RRUR 5 UR b*[RRUR 5 ggURSURS-UR-nX0R-n[RR UR S[R"S U- 55 UR b*[RRUR 5 ggg) z&Initialize convolution module weights.rrrNrrrr-g@)rrrrr r rHr r#r rt out_channelsgroupsmathsqrt)rrrfan_outs r^ _init_convr4sV&"))$$ X  GGOOFMMsO 3{{&v{{+' ~ % V]] 4{{&v{{+'  & GG " "6== 1{{&v{{+'((+f.@.@.CCfFYFYYG  %G GGOOFMM1diig .F G{{&v{{+'%%raruchannelsc.U(dgX-S:XdeX-$)z3Calculate number of groups for grouped convolution.r-rrm)rur5s r^ num_groupsr7s$ $)))%%rac ^\rSrSrSrSS\"5SSS4S\S\S \S \\\4S \S \4 U4S jjjr SS\ SS4Sjjr S\ RS\ R4SjrSrU=r$) MbConvBlockizGPre-Norm Conv Block - 1x1 - kxk - 1x1, w/ inverted bottleneck (expand).r-r-r-rCNin_chsout_chsrdilationrrc :>XxS.n [TU]5 [[URUR 5UR S9n [UR(aUOUUR-5n [URU 5n US:Xa4[X4URURURS.U D6UlO["R$"5UlUR&S;deSupnUR&S:XaX4Snn OUR&S :XaX4SnnOX4S nnU "U4S UR(0U D6UlU S:a)[X4UR,URS .U D6UlO["R$"5Ul[1XS4S U0U D6UlU "U 40U D6Ul[1U U UR64UUU URS.U D6Ul0n[;UR<[>5(a]UR<S:XdUR<S:Xa=UR@US'[CURDUR(aUOU -5US'URF(a4[IUR<U 40UDU D6Ul%U "U 40U D6Ul&SUl'O3SUl%U "U 40U D6Ul&[IUR<U 40UDU D6Ul'[1XS4SUR0U D6Ul(US:a[SU5Ul*g["R$"5Ul*g)z Args: in_chs: Input channels. out_chs: Output channels. stride: Stride for conv. dilation: Dilation for conv. cfg: Convolution block configuration. drop_path: Drop path rate. rrr)rGrHr|)r1x1rx)r-r-r-rr-r?r apply_act)rGr|r)rr=r0r|r~ecarS rd_channelsNrHrC)+rrrr rUrSrYr&rrr?r7rurrGrwr|rrrryrvpre_normrzrr! conv1_1x1rrt conv2_kxkrrrkrrgrr}rse_earlyr!r~ conv3_1x1rr)r]r;r<rr=rrrrrnorm_act_layermid_chsr0 stride_poolstride_1stride_2 dilation_2 attn_kwargsrs r^rMbConvBlock.__init__s( /  !3CNNCMM!RX[XdXde S->->'FcN^N^!^_CNNG4 Q;(k+.==sX[XcXckgikDMKKMDM"7777*1' x ??f $&,qkK __ %#)A;jHj#)A;jH&vP9I9IPRP ?$Vss?W?WadalalsprsDI DI&vQ(QbQ#G2r2 &   OO  KK     cnnc * *~~%5)@+.+=+= K(-0cN_N_7el1m-n M* >>'U;URTUDM'626DJDG DM'626DJ!#..'O[OBODG&wWWTVW09B),BKKMrarrc4[[[US9U5 gr&r1rr4r(s r^r)MbConvBlock.init_weightseGJv6=rarcURU5nURU5nURU5nURU5nUR U5nUR U5nUR bUR U5nURU5nURbURU5nURU5nURU5U-nU$r[) rrCrrDrrErFr!r~rGrr]rrs r^rMbConvBlock.forwardhs==# MM!  IIaL NN1  JJqM NN1  == $ a A JJqM 77  A NN1  NN1  (ra) rDrErGrrrr!rCr~rFrr-)rbrcrdrerfr6rgrrjrrkr)rrrrlrrs@r^r9r9sQ (."0"2!PRPRPR PR CHo PR  PRPRPRd>3>>%,,rar9c^\rSrSrSrSSSS\"5SSSS4 S \S \\S \S \S \\\4S\S\ S\ 4U4Sjjjr S\ RS\ R4SjrSrU=r$) ConvNeXtBlockizConvNeXt Block.Nr-r:TrCr;r<rtrr=rconv_mlprc >XS.n [TU]5 U=(d Un[UR5n U(a.[ [ UR 5URS9n [nOSUR ;de[n [nXpl US:Xa[X40U D6Ul OIX:wa*[R"X4SUR S.U D6Ul O[R""5Ul UR$S;deSunnUR$S :XaUnOUnUS:Xa[X4S UR&0U D6UlO[R""5Ul[+UU4UUUSS UR S .U D6UlU "U40U D6UlU"U[1UR2U-54UR U S .U D6UlU(aBUR6(a[9X&R640U D6O[R""5UlOAUR6(a[=X&R640U D6O[R""5UlUS:a[?U5Ul g[R""5Ul g)a) Args: in_chs: Input channels. out_chs: Output channels. kernel_size: Kernel size for depthwise conv. stride: Stride for conv. dilation: Dilation for conv. cfg: Convolution block configuration. conv_mlp: Whether to use convolutional MLP. drop_path: Drop path rate. rrrVrr-)rtrH)rrxr:rrGT)rtrr= depthwiserH)rHrSrCN)!rrrrSrrrUrYrrr use_conv_mlprrrrrwrryrzrr!conv_dwrrgr?rrQrlsrrr)r]r;r<rtrr=rrZrrrrrSrU mlp_layerrJ stride_dwrs r^rConvNeXtBlock.__init__s0 / #V!#--0  !?S\\RJI#..0 00"JI$ Q;(?B?DM  IIfa13??a^`aDMKKMDM.000!% Y ??f $ KI ! $V^s?W?W^[]^DI DI$    $a[     w-"-     7* +      FIool7OOBrB[][f[f[hDGDGOOj//@R@Y[YdYdYfDG09B),BKKMrarrcURU5nURU5nURU5nUR(a4UR U5nUR U5nUR U5nO[URSSSS5nUR U5nUR U5nUR U5nURSSSS5nURU5U-nU$Nrrrsr-) rrr^r]rrr_permuterrUs r^rConvNeXtBlock.forwards==# IIaL LLO    ! A A A !Q1%A ! A A A !Q1%A NN1  (ra)r^rrr_rrrr])rbrcrdrerfr6rgr rrirjrrrrrlrrs@r^rXrXs &* (."0"2!!ORORc]OR OR  OR CHo OR ORORORORb%,,rarXrrMc`URup#pE[X1S-S:HSUSUSS35 [XAS-S:HSUSUSS35 URX#US-USXAS-USU5nURSSSSS S 5R 5RS USUSU5nU$) z'Partition into non-overlapping windows.rheight () must be divisible by window ()r-width (rsrrrrr'rrer)rrMrrrrwindowss r^window_partitionrosJA! AA ! #xs2QR]^_R`Qaab%cd AA ! #wqc1PQ\]^Q_P``a%bc q{1~%{1~qN7JKXYN\]^Aii1aAq)446;;B AP[\]P^`abG Nrarnimg_sizecUup4URSnURSX1S-XAS-USUSU5nURSSSSSS5R5RSX4U5nU$)zReverse window partition.rrr-rsrrlrrrrerrnrMrprrrrs r^window_reverserts DA bA R!n,aq>.A;q>S^_`SacdeA !Q1a#..055b!BA HrarNc ZURup#pE[X1S-S:HSUSUS35 [XAS-S:HSUSUS35 URX!SX1S-USXAS-U5nURSSSSSS 5R 5RS USUSU5nU$) z6Partition into overlapping windows with grid striding.rheight  must be divisible by grid r-width rrlrsrrrm)rrNrrrrrns r^grid_partitionrysJA! A!  !WQC/J9UV<.#YZ A!  !VA3.I)TU,#XY qA,q\ 19Q<PQlARTUVAii1aAq)446;;B ! iXYl\]^G NracUup4URSnURSX1S-XAS-USUSU5nURSSSSSS5R5RSX4U5nU$)zReverse grid partition.rrr-rsrlrrrrrnrNrprrrrs r^ grid_reverser|s~ DA bA Rl*A1,=y|YWX\[\]A !Q1a#..055b!BA HrarcSnURS:Xa[[XRS9nU$URS:Xa[[US9nU$URS:Xa[[ US9nU$)z,Get relative position class based on config.Nr)rM hidden_dimrH)rMbias_tf)rIrr(rKr)r*)rrMrs r^get_rel_pos_clsrssK 5 i[__]   V #jkB    Y &l D rac d^\rSrSrSrS\"5SSS4S\S\S\S \4U4S jjjr S r S r S r U=r $)PartitionAttentionClizJGrid or Block partition + Attn + FFN. NxC 'channels last' tensor layout. blockrCNrpartition_typerrc H>XVS.n[T U]5 [[UR5UR S9n[ UR5n US:HUl[UR(a URO UR5Ul [X0R5n U"U40UD6Ul[UU4UR UR"UR$U UR&UR(S.UD6UlUR,(a[/U4SUR,0UD6O[0R2"5UlUS:a [7U5O[0R2"5UlU"U40UD6Ul[=SU[?XR@-5U UR(S.UD6Ul!UR,(a[/U4SUR,0UD6O[0R2"5Ul"US:a[7U5Ul#g[0R2"5Ul#g) Nrrrr<rHr=rrDrErQrCrrm)$rrrrrWrYrrSpartition_blockr$rMrNpartition_sizerrrr<rBr=rDrErrQrrrrrr r!rrgr?rr"r# r]rrrrrrrrUrSrrs r^rPartitionAttentionCl.__init__ s / ^C,=,=>CLLQ !#--0 -8'4;O;OUXUbUbc%c+>+>? *r*     \\~~#mmmm     JM:cEsE"E^`^i^i^k1:R(9-R[[]*r*  &6&6 67    JM:cEsE"E^`^i^i^k1:R(9-R[[]rac<URSSnUR(a[XR5nO[ XR5nUR U5nUR(a[ X0RU5nU$[X0RU5nU$)Nr-rs)rrrorryrrtr|r]rrp partitioneds r^_partition_attn$PartitionAttentionCl._partition_attnLs771Q<   *1.A.ABK(,?,?@Kii ,   {,?,?JA[*=*=xHArac XRURURURU5555-nXR UR UR URU5555-nU$r[r rrrr#r"rr!rs r^rPartitionAttentionCl.forward[_ )=)=djjm)L MN N $**Q-)@ AB Bra rr r#rr"rrr!rr)rbrcrdrerfr7rgrkrjrrrrlrrs@r^rrsb#*)>)@!*S*S *S' *S  *S*SX rarc^\rSrSrSr\"5SSS4S\S\S\4U4SjjjrS \ RS \ R4S jr S \ RS \ R4S jr S r U=r$)ParallelPartitionAttentioniazIExperimental. Grid and Block partition + single FFN. NxC tensor layout. rCNrrrc >XES.n[T U]5 US-S:Xde[[UR5UR S9n[ UR5nURUR:Xde[UR5Ul [X R5n U"U40UD6Ul [UUS-4URUR UR"U UR$UR&S.UD6Ul[UUS-4URUR UR"U UR$UR&S.UD6UlUR,(a[/U4SUR,0UD6O[0R2"5UlUS:a [7U5O[0R2"5UlU"U40UD6Ul[=S U[?XR@-5UUUR&S.UD6Ul!UR,(a[/U4SUR,0UD6O[0R2"5Ul"US:a[7U5Ul#g [0R2"5Ul#g ) zj Args: dim: Input dimension. cfg: Transformer block configuration. drop_path: Drop path rate. rrrrrrQrC)rr out_featuresrSrNrm)$rrrrrWrYrrSrMrNr$rrrrr<rBr=rDrE attn_block attn_gridrQrrrrrr r!rrgr?rr"r#) r]rrrrrrrUrSrrs r^r#ParallelPartitionAttention.__init__gs / Qw!||^C,=,=>CLLQ !#--0 #--///'8%c+>+>? *r* %  1H  \\~~#mmmm    %  1H  \\~~#mmmm    JM:cEsE"E^`^i^i^k1:R(9-R[[]*r*  &6&6 67    JM:cEsE"E^`^i^i^k1:R(9-R[[]rarrc<URSSn[XR5nURU5n[ X0RU5n[ XR5nUR U5n[XPRU5n[R"XF/SS9$)Nr-rsrr) rrorrrtryrr|rcat)r]rrppartitioned_blockx_windowpartitioned_gridx_grids r^r*ParallelPartitionAttention._partition_attns771Q<,Q0C0CD OO,=>!"35H5H(S)!-@-@A>>*:;.0C0CXNyy(+44rac XRURURURU5555-nXR UR UR URU5555-nU$r[rrs r^r"ParallelPartitionAttention.forwardrra) rrr r#rr"rrr!r)rbrcrdrerfr7rgrjrrrrrrlrrs@r^rras*?)@! <S<S'<S <S<S| 5 5%,, 5%,,rarc \URup#pE[XAS-S:HSUSUSS35 [XQS-S:HSUSUSS35 URX#XAS-USXQS-US5nURSSSSS S 5R 5RS X1SUS5nU$) z#Partition windows for NCHW tensors.rrhrirjr-rkrrlrsrrrm)rrMrrrrrns r^window_partition_nchwrsJA! AA ! #xs2QR]^_R`Qaab%cd AA ! #wqc1PQ\]^Q_P``a%bc qQa.(+a.!1~:M{[\~^Aii1aAq)446;;Bq>S^_`SabG Nrac Uup4URSnURSX1S-XAS-XQSUS5nURSSSSSS5R5RSXSU5nU$)z*Reverse window partition for NCHW tensors.r-rrrsrlrrrrrss r^window_reverse_nchwrs~ DA aA R!n,aq>.A1RSnVabcVdeA !Q1a#..055b!BA Hrac XURup#pE[XAS-S:HSUSUS35 [XQS-S:HSUSUS35 URX#USXAS-USXQS-5nURSSSSSS 5R 5RS X1SUS5nU$) z Grid partition for NCHW tensors.rrvrwr-rxrsrrrlrrm)rrNrrrrrns r^grid_partition_nchwrsJA! A!  !WQC/J9UV<.#YZ A!  !VA3.I)TU,#XY qYq\1! #4ilASTDUVAii1aAq)446;;BQXVS.n[T U]5 [[UR5UR S9n[ UR5n US:HUl[UR(a URO UR5Ul [X0R5n U"U40UD6Ul[UU4UR UR"UR$U UR&UR(S.UD6UlUR,(a[/U4SUR,0UD6O[0R2"5UlUS:a [7U5O[0R2"5UlU"U40UD6Ul[=S U[?XR@-5U UR(S.UD6Ul!UR,(a[/U4SUR,0UD6O[0R2"5Ul"US:a[7U5Ul#g[0R2"5Ul#g) z Args: dim: Input dimension. partition_type: Partition type ('block' or 'grid'). cfg: Transformer block configuration. drop_path: Drop path rate. rrrrrQrCrNrm)$rrrrrUrYrrSrr$rMrNrrrrr<rBr=rDrErrQrrrrrr r!rrgr?rr"r#rs r^rPartitionAttention2d.__init__s  / ^CNN;N !#--0 -8'4;O;OUXUbUbc%c+>+>? *r*     \\~~#mmmm     LO??<GGBG`b`k`k`m1:R(9-R[[]*r*  &6&6 67    LO??<GGBG`b`k`k`m1:R(9-R[[]rarrc<URSSnUR(a[XR5nO[ XR5nUR U5nUR(a[ X0RU5nU$[X0RU5nU$)Nr)rrrrrrrrrs r^r$PartitionAttention2d._partition_attns7723<   /3F3FGK-a1D1DEKii ,   #K1D1DhOA"+/B/BHMArac XRURURURU5555-nXR UR UR URU5555-nU$r[rrs r^rPartitionAttention2d.forward(rrar)rbrcrdrerfr7rgrkrjrrrrrrlrrs@r^rrs#*)>)@!1S1S 1S' 1S  1S1Sf  %,, %,,rarc |^\rSrSrSrS\"5\"5SSS4S\S\S\S \S \S \4 U4S jjjr SS jr Sr Sr U=r $) MaxxVitBlocki.z;MaxVit conv, window partition + FFN , grid partition + FFN.r-rCNrrrrrrc Z>XxS.n [T U]5 URUlURS:Xa[ O[ n U "X4X4US.U D6Ul[SX%US.U D6n UR(a[O[n UR(aSOU "S0U D6Ul U "SSS0U D6Ul g) aInitialize MaxxVitBlock. Args: dim: Input channel dimension. dim_out: Output channel dimension. stride: Stride for downsampling. conv_cfg: Configuration for convolutional blocks. transformer_cfg: Configuration for transformer blocks. drop_path: Drop path rate. rrrrrrrrNrgridrm)rrrP nchw_attnrqrXr9convdictrrrOrr)r]rrrrrrrrrconv_clsrNpartition_layerrs r^rMaxxVitBlock.__init__1s* / (66$,$7$7:$E=;Sb&R[b_ab WwyWTVW 26...FZ"1"?"?$_EcWbEc(NN+NracURb"[[[US9UR5 [[[US9UR5 [[[ US9UR 5 gr&)rr1rrrr4rr(s r^r)MaxxVitBlock.init_weightsRsM ?? &  1&A4?? SG-f=t~~NGJv6 BracURU5nUR(dURSSSS5nURbURU5nUR U5nUR(dURSSSS5nU$rd)rrrerrrs r^rMaxxVitBlock.forwardXsp IIaL~~ !Q1%A ?? &"A NN1 ~~ !Q1%Ara)rrrrr-)rbrcrdrerfr6r7rgrjrr)rrlrrs@r^rr.s}E '5'75J5L!OOO O % O 3 OOOBC  rarc^\rSrSrSrSS\"5\"5SSS4S\S\S \S \S \S \S \4U4Sjjjr SS\ SS4Sjjr S\ RS\ R4SjrSrU=r$)ParallelMaxxVitBlockifzQMaxVit block with parallel cat(window + grid), one FF. Experimental timm block. r-rrCNrrrnum_convrrrc *>XS.n [T U]5 URS:Xa[O[n US:a:U "X4X5US.U D6/n X"X"4XWS.U D6/US- -- n [ R "U 6UlOU "X4X5US.U D6Ul[SX&US.U D6Ul g) a Args: dim: Input dimension. dim_out: Output dimension. stride: Stride for first conv block. num_conv: Number of convolution blocks. conv_cfg: Convolution block configuration. transformer_cfg: Transformer block configuration. drop_path: Drop path rate. rrr-r)rrrNrm) rrrqrXr9rrrrr)r]rrrrrrrrrrrconvsrs r^rParallelMaxxVitBlock.__init__ls, / $,$7$7:$E=; a<cc6S\c`bcdE hwYXYVXYZ^fij^jk kE u-DI ffV_fcefDI.k7[dkhjk rarrc[[[US9UR5 [[[US9UR 5 gr&)r1rrrr4rr(s r^r)!ParallelMaxxVitBlock.init_weightss-G-f=tyyIGJv6 BrarcURU5nURSSSS5nURU5nURSSSS5nU$rd)rrerrs r^rParallelMaxxVitBlock.forwardsI IIaL IIaAq ! IIaL IIaAq !ra)rrr-)rbrcrdrerfr6r7rgrjrrkr)rrrrlrrs@r^rrfs'5'75J5L! l l l l  l % l3 l l lDC3CC%,,rarc^\rSrSrSrSSSS\"5\"5SSS4 S \S \S \S \S \\\4S\ \ \\ 4S\S\S\ \ \ \ 44U4Sjjjr S\RS\R4SjrSrU=r$) MaxxVitStageizEMaxxVit stage consisting of mixed convolution and transformer blocks.rrl)rrrCNr;r<rdepth feat_size block_typesrrrc >XS.n [TU]5 SUl[Xd5n/n [ U5HupUS:XaUOSnUS;deUS:Xa1UR S:Xa[ O[nU U"UU4UUXS.U D6/- n OgUS :Xa$[Xu5nU [UU4UUUXS .U D6/- n O=US :XaU [UU4UUUXS .U D6/- n OUS :XaU [UU4UUUXS .U D6/- n UnM [R"U 6Ulg)az Args: in_chs: Input channels. out_chs: Output channels. stride: Stride for first block. depth: Number of blocks in stage. feat_size: Feature map size. block_types: Block types ('C' for conv, 'T' for transformer, etc). transformer_cfg: Transformer block configuration. conv_cfg: Convolution block configuration. drop_path: Drop path rate(s). rFrr-)rrMPMrrrr)rrrrr)rrrrrN)rrgrad_checkpointingr% enumeraterqrXr9rrrrrrblocks)r]r;r<rrrrrrrrrrrit block_striderrrs r^rMaxxVitStage.__init__s4 / "'";6 k*DA%&!V6L-- --Cx,4,?,?:,M=S^8( 'l   c-oI -( +''l c<(%$3'l d/(%$3'l FY+ZmmV, rarrcUR(a;[RR5(d[ UR U5nU$UR U5nU$r[)rrjit is_scriptingr2rrs r^rMaxxVitStage.forwardsH  " "599+A+A+C+Ct{{A.A AAra)rr)rbrcrdrerfr7r6rgrrrkrjr rrrrrlrrs@r^rrsO )1255J5L'5'735M-M-M- M-  M- S#X M-sE#J/M-3M-%M-UDK/0M-M-^%,,rarc^\rSrSrSrSS\S\S\S\S\S \S \S \4U4S jjjr SS \SS4Sjjr S\ RS\ R4Sjr SrU=r$)Stemiz"Stem layer for feature extraction.Nr;r<rtr|rHrSrUrYc H>XS.n [T U]5 [U[[45(d [ U5n[ [Xv5US9n USUlSUl [XSU4SXES.U D6Ul U "US40U D6Ul [USUSU4SXES.U D6Ul g) a Args: in_chs: Input channels. out_chs: Output channels. kernel_size: Kernel size for convolutions. padding: Padding mode. bias: Whether to use bias. act_layer: Activation layer. norm_layer: Normalization layer. norm_eps: Normalization epsilon. rrrrr)rr|rHr-N)rrrlisttupler$rr r<rr!conv1rconv2)r]r;r<rtr|rHrSrUrYrrrrHrs r^r Stem.__init__s0 / 'D%=11(G !3J!JPXYr{  "61:{o1V]olno #GAJ5"5 "71:wqz;sqZasprs rarrc4[[[US9U5 gr&rQr(s r^r)Stem.init_weightsrSrarclURU5nURU5nURU5nU$r[)rrrrs r^r Stem.forwards. JJqM JJqM JJqMra)rrrr<r)rsr{FrRrrNNr-)rbrcrdrerfrgrkrirjrr)rrrrlrrs@r^rrs, !#+"#t#t#t #t  #t  #t#t#t#t#tJ>3>>%,,rarcURbUR(deU$USUR-USUR-4n[XUS9nU$)z>Configure window size based on image size and partition ratio.rr-)rMrN)rMrNrLr)rrprs r^cfg_window_sizer&sV "}}} a[C$7$77!H[H[9[[N #^ LC Jrakwargsc j0n0n0nUR5HaupVURS5(aXbURSS5'M1URS5(aXcURSS5'M]XdU'Mc [U4[UR40UD6[UR40UD6S.UD6nU$)z-Overlay keyword arguments onto configuration. transformer_r{conv_)rr)items startswithrrr)rrtransformer_kwargs conv_kwargs base_kwargsrrs r^_overlay_kwargsr0sKK  << ' '@Aqyy< = \\' " "23 '2. /N     3 3J7IJ55    C Jrac^\rSrSrSrS)S\S\\\\\44S\S\S\ S \ S \ S \ 4U4S jjjr S*S \ RS\ S\ SS4Sjjr\R"R$S\\ 4Sj5r\R"R$S+S\S\\ \ 44Sjj5r\R"R$S,S\SS4Sjj5r\R"R$S\ R4Sj5rS-S\S\\ SS4SjjrS.S\R8S\\\\\4S\S\S\ S\S\\\R8\\R8\\R8444SjjrS/S\\\\4S \S!\S\\S"44S#jjrS\R8S\R84S$jr S+S\R8S%\S\R84S&jjr!S\R8S\R84S'jr"S(r#U=r$$)0r8iEzsCoaTNet + MaxVit base model. Highly configurable for different block compositions, tensor layouts, pooling types. Nrrpin_chans num_classes global_pool drop_ratedrop_path_raterc >[TU]5 XS.n [U5nU (a [U40U D6n[ UR U5n X@lX0lXPlURS=Ul Ul X`l SUl /Ul [SUURUR R"UR$UR R&UR R(UR R*S.U D6UlUR,R.n U=R[1UR,R2SSS9/- sl [5[7U[U 55VVs/sH upX-PM snn5n[9UR5n[9UR:5U:Xde[=XqR:SS 9nUR,R2n/n[?U5HnSnURUn[5UVs/sHnUS - U-S -PM sn5nU[AUU4UR:UURBUUR U UUUS .U D6/- nU U-n UnU=R[1UU S U3S9/- sl M [DRF"U6Ul$[K[MUR R(5UR R*S 9nURN(aW[DRP"5Ul)URNUl'[UURU4URNUUUS.U D6Ul+OHURUl'U"UR40U D6Ul)[YURU4UUS.U D6Ul+URZS;deURZ(a)[][KUR^URZS9U5 ggs snnfs snf)aH Args: cfg: Model configuration. img_size: Input image size. in_chans: Number of input channels. num_classes: Number of classification classes. global_pool: Global pooling type. drop_rate: Dropout rate. drop_path_rate: Drop path rate. **kwargs: Additional keyword arguments to overlay on config. rrF)r;r<r|rHrSrUrYrstem)num_chs reductionrT) stagewiser-)rrrrrrzstages.r) hidden_sizerGrrU)rGr)r{rrrrr'Nrm)0rrr$rrrrrrr num_featuresrr feature_inforrrr|rrSrUrYrrrr<rziplenrrrangerrqrrstagesrrrrrrheadrrr1 _init_weights)r]rrprrrrrrrrrrrrsr num_stagesdprr;r  stage_strider<rfinal_norm_layerrs r^rMaxxVit.__init__Ks0  /X& !#00C)#*=*=xH& &-0]]2->>DN""'  NNLL((ll,,||..\\**     !! d499+<+<RXYZZc(If#2E2E2P2P#QWZWjWjWsWst    DI$'$8$8D !-!!!11%#+ DI%)$5$5D !():):AbADI&!!&#   DI"\\\\ ??  2 23??KT R mPOs +O O% rrrrc[US5(aURUS9 gg![a UR5 gf=f)Nr)r')hasattrr) TypeError)r]rrrs r^rMaxxVit._init_weightssG 6> * * &##6#2 + &##% &s$AAc^UR5V^Vs1sH$umn[U4SjS55(dM"TiM& snn$s snnf)Nc3,># UH oT;v M g7fr[rm).0nrs r^ *MaxxVit.no_weight_decay..sS#Ra6#Rs)relative_position_bias_tablez rel_pos.mlp)named_parametersany)r]r_s ` r^no_weight_decayMaxxVit.no_weight_decaysJ//1U1$!QS#RSS 1U UUs !AAcoarsec[SSS/S9nU$)Nz^stem)z^stages\.(\d+)N)z^norm)i)rr)r)r]r&matchers r^ group_matcherMaxxVit.group_matchers-/CD raenablec6URH nXlM gr[)r r)r]r+rs r^set_grad_checkpointingMaxxVit.set_grad_checkpointingsA#) rac.URR$r[)r fcr\s r^get_classifierMaxxVit.get_classifiersyy||racFXlURRX5 gr[)rr reset)r]rrs r^reset_classifierMaxxVit.reset_classifiers&  1rarindicesr stop_early output_fmtintermediates_onlycPUS;dS5e/n[[UR5S-U5upSn URU5nX;aUR U5 [UR5n [ R R5(dU(d URn OURSU n U HHn U S- n U "U5nX;dMU(aX:XaURU5nOUnUR U5 MJ U(aU$X:XaURU5nX4$)aForward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.r-rN) r/r r rappendrrrr)r]rr7rr8r9r: intermediates take_indices max_indexfeat_idxlast_idxr stagex_inters r^forward_intermediatesMaxxVit.forward_intermediatess*Y&D(DD& "6s4;;7G!7KW"U  IIaL  #   #t{{# 99 ! ! # #:[[F[[),FE MHaA'H0"iilGG$$W-    ! Ara prune_norm prune_head.c[[UR5S-U5upEURSUUlU(a[R"5UlU(aUR SS5UlU$)z6Prune layers not required for specified intermediates.r-Nrr{)r/r r rrrr5r )r]r7rGrHr?r@s r^prune_intermediate_layers!MaxxVit.prune_intermediate_layerss`#7s4;;7G!7KW"U kk*9-  DI --a4DIraclURU5nURU5nURU5nU$r[)rr rrs r^forward_featuresMaxxVit.forward_featuress. IIaL KKN IIaLra pre_logitscPU(aURXS9$URU5$)N)rO)r )r]rrOs r^ forward_headMaxxVit.forward_head#s#6@tyyy2RdiiPQlRracJURU5nURU5nU$r[)rMrQrs r^rMaxxVit.forward&s'  ! !! $   a ra) rrrrrr rrrrrr r)rsrrCrCNNr-F)Tr[)NFFr<F)r-FT)%rbrcrdrerfr5rrgrrkrjrrrModulerrrignorer r$rir r)r-r1r r5rr rErJrMrQrrlrrs@r^r8r8Es58#$!$&iSiSCsCx01iS iS  iS  iSiS"iSiSiSV&BII&S&#&t& YYUSUU  YYDT#s(^ YY*T*T** YY 2C2hsm2W[28<$$',4 ||4 eCcN344  4  4  4 !%4  tELL!5tELL7I)I#JJ K4 p./$# 3S >*    sCx %,,5<< SellSSS%,,rar8rFTrHrJryrGconv_output_biasconv_attn_earlyconv_attn_act_layerconv_norm_layertransformer_shortcut_biastransformer_norm_layertransformer_norm_layer_clrQrIrKc P[[UUSSUUUSUS9 [SUUU UUU U S9S9$)a;RW variant configuration for CoAtNet models. These models were created and trained before seeing https://github.com/google-research/maxvit Common differences for initial timm models: - pre-norm layer in MZBConv included an activation after norm - mbconv expansion calculated from input instead of output chs - mbconv shortcut and final 1x1 conv did not have a bias - SE act layer was relu, not silu - mbconv uses silu in timm, not gelu - expansion in attention block done via output proj, not input proj Variable differences (evolved over training initial models): - avg pool with kernel_size=2 favoured downsampling (instead of maxpool for coat) - SE attention was between conv2 and norm/act - default to avg pool for mbconv downsample instead of 1x1 or dw conv - transformer block shortcut has no bias TFr) ryrGrvrrrwr}rrSrU)r@rArGrQrUrWrIrKrrrr6r7) ryrGrZr[r\r]r^r_r`rQrIrKs r^ _rw_coat_cfgrd,sW@ #(&.&  .3#-3%#   rarxconv_attn_ratior<c N[[UUSUUSUS9[SUUUU UUU U S9 S9$)aRW variant configuration for MaxViT models. These models were created and trained before seeing https://github.com/google-research/maxvit Differences of initial timm models: - mbconv expansion calculated from input instead of output chs - mbconv shortcut and final 1x1 conv did not have a bias - mbconv uses silu in timm, not gelu - expansion in attention block done via output proj, not input proj Fr)ryrGrrrwrrSrU) r@rGr<rMrQrUrWrIrKrbrc) ryrGrZrer]r_r`rMr<rQrIrKs r^ _rw_max_cfgrgesS0 #(&& .##-3%#   rarXrconv_norm_layer_clrOc p[U5n[[SUUSUSUUS9[SUUUUSUUU U S9 S9$)z=Configuration for experimental ConvNeXt-based MaxxViT models.rFr)rqryrGrrrQrUrWr-) r@rGrMrOrQrUrWrIrKrb)r$rr6r7) ryrGr]rhr_r`rMrOrQrIrKs r^ _next_cfgrjseK(K !##A&, .#'#A-3%#   rac <[[SSSS9[SSSSS9S 9$) z0Configuration matching TensorFlow MaxViT models.gMbP? gelu_tanhsame)rYrSr|rFr)rYrSr=rIrbrcrmrar^_tf_cfgrns6 ! .!"   racoatnet_pico_rw)rrJr)r;r)rrrr)rZrecoatnet_nano_rw)rsrlrs)ryrZre coatnet_0_rwr)rrsrYr)r[r^ coatnet_1_rw)rrsrr)ryr[r^ coatnet_2_rw)rprqrJ)rrpr)ryr\ coatnet_3_rw)rrr)rr)ryr\rQcoatnet_bn_0_rwr)ryr[r^r_coatnet_rmlp_nano_rwr)rZrerIrKcoatnet_rmlp_0_rw)ryrIcoatnet_rmlp_1_rwr)rGr[r^rIrKcoatnet_rmlp_1_rw2)ryrIrKcoatnet_rmlp_2_rw)ryr\rQrIcoatnet_rmlp_3_rwcoatnet_nano_cc)rrrrr)rrrrqcoatnext_nano_rwr)rrrr)rN)rIrQ coatnet_0rr)rrrr coatnet_1 coatnet_2rprw coatnet_3rry coatnet_4)r r coatnet_5)rqrJrmaxvit_pico_rw)r;rrprq)rrrr)rrrr)r;)rrrqrmaxvit_nano_rw)r-rrsr-maxvit_tiny_rwmaxvit_tiny_pm)rrrrmaxvit_rmlp_pico_rw)rImaxvit_rmlp_nano_rwmaxvit_rmlp_tiny_rwmaxvit_rmlp_small_rwmaxvit_rmlp_base_rw)rrrqrrmaxxvit_rmlp_nano_rw)rrrqrrmaxxvit_rmlp_tiny_rwmaxxvit_rmlp_small_rw)0rmaxxvitv2_nano_rw)rOrImaxxvitv2_rmlp_base_rw)rrsrr)rOmaxxvitv2_rmlp_large_rw)i@ir)rrsr)Prrmaxvit_tiny_tf)rrrqrrrmaxvit_small_tfmaxvit_base_tfmaxvit_large_tfmaxvit_xlarge_tf state_dictmodelc~UR5n0nUR5GHupEURS5(aURUSS5nURUR R:wd UR SUR S:wa)[UUR UR RS9nXB;aoURX$R:waSUR5X$R5:Xa/URS;deURX$R5nXSU'GM U$)z/Filter checkpoint state dict for compatibility.r Nirr-)new_window_sizenew_bias_shape)rrl) rrendswith get_submodulerr rMr,ndimnumelr)rrmodel_state_dictout_dictrrms r^checkpoint_filter_fnr>s'')H  " ::4 5 5##AdsG,Aww!88>>>!--PQBRVWVcVcdeVfBf-$%MM#$#A#A#G#G QVV/?/B/G/G%GAGGIYiYlYrYrYtLt66V# ## *-334A # Oravariant cfg_variant pretrainedc Uc0U[;aUnO#SRURS5SS5n[[X4[U[ SS9[ S.UD6$)zCreate a MaxxVit model variant.Nr#rT)flatten_sequential) model_cfg feature_cfgpretrained_filter_fn) model_cfgsjoinsplitr.r8rr)rrrrs r^_create_maxxvitrTsg j !K((7==#5cr#:;K  [)D11     raurlc $USSSSSSSSSS S S . UE$) z$Create a default configuration dict.rV)rsrUrU)rYrYffffff?bicubic)?rrz stem.conv1zhead.fcTz apache-2.0) rr input_size pool_sizecrop_pct interpolationmeanr first_conv classifierfixed_input_sizelicenserm)rrs r^_cfgrcs74}SY9")    $*  razcoatnet_pico_rw_224.untrained)rzcoatnet_nano_rw_224.sw_in1kztimm/zyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_nano_rw_224_sw-f53093b4.pthg?) hf_hub_idrrzcoatnet_0_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_0_rw_224_sw-a6439706.pth)rrzcoatnet_1_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_1_rw_224_sw-5cae1ea8.pthz!coatnet_2_rw_224.sw_in12k_ft_in1k)rz'coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_384.sw_in12k_ft_in1k)rsrr)rrg?squash)rrrr crop_modezcoatnet_bn_0_rw_224.sw_in1kzyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_bn_0_rw_224_sw-c228e218.pthr)rrrrrz coatnet_rmlp_nano_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_nano_rw_224_sw-bd1d51b3.pthzcoatnet_rmlp_0_rw_224.untrainedzcoatnet_rmlp_1_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_1_rw_224_sw-9051e6c3.pthzcoatnet_rmlp_2_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_2_rw_224_sw-5ccfac55.pthzcoatnet_rmlp_3_rw_224.untrainedzcoatnet_nano_cc_224.untrainedzcoatnext_nano_rw_224.sw_in1kzzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnext_nano_rw_224_ad-22cb71c2.pthzcoatnet_2_rw_224.sw_in12ki-.)rrzcoatnet_3_rw_224.sw_in12kzcoatnet_rmlp_1_rw2_224.sw_in12kzcoatnet_rmlp_2_rw_224.sw_in12kzcoatnet_0_224.untrainedzcoatnet_1_224.untrainedzcoatnet_2_224.untrainedzcoatnet_3_224.untrainedzcoatnet_4_224.untrainedzcoatnet_5_224.untrainedzmaxvit_pico_rw_256.untrained)rsrqrq)r)rrrzmaxvit_nano_rw_256.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-fb127241.pth)rrrrzmaxvit_tiny_rw_224.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_tiny_rw_224_sw-7d0dffeb.pthzmaxvit_tiny_rw_256.untrainedzmaxvit_tiny_pm_256.untrainedzmaxvit_rmlp_pico_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_pico_rw_256_sw-8d82f2c6.pthzmaxvit_rmlp_nano_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_nano_rw_256_sw-c17bb0d6.pthzmaxvit_rmlp_tiny_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_tiny_rw_256_sw-bbef0ff5.pthz maxvit_rmlp_small_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_small_rw_224_sw-6ef0ae4f.pthz"maxvit_rmlp_small_rw_256.untrainedz(maxvit_rmlp_base_rw_224.sw_in12k_ft_in1kz(maxvit_rmlp_base_rw_384.sw_in12k_ft_in1kz maxvit_rmlp_base_rw_224.sw_in12kz maxxvit_rmlp_nano_rw_256.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_nano_rw_256_sw-0325d459.pthz"maxxvit_rmlp_tiny_rw_256.untrainedz!maxxvit_rmlp_small_rw_256.sw_in1kzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_small_rw_256_sw-37e217ff.pthzmaxxvitv2_nano_rw_256.sw_in1k)rrrz+maxxvitv2_rmlp_base_rw_224.sw_in12k_ft_in1kz+maxxvitv2_rmlp_base_rw_384.sw_in12k_ft_in1kz%maxxvitv2_rmlp_large_rw_224.untrainedz#maxxvitv2_rmlp_base_rw_224.sw_in12kzmaxvit_tiny_tf_224.in1k)rrrzmaxvit_tiny_tf_384.in1kzmaxvit_tiny_tf_512.in1k)rsrJrJ)rrzmaxvit_small_tf_224.in1kzmaxvit_small_tf_384.in1kzmaxvit_small_tf_512.in1kzmaxvit_base_tf_224.in1kzmaxvit_base_tf_384.in1kzmaxvit_base_tf_512.in1kzmaxvit_large_tf_224.in1kzmaxvit_large_tf_384.in1kzmaxvit_large_tf_512.in1kzmaxvit_base_tf_224.in21kiSUz maxvit_base_tf_384.in21k_ft_in1kz maxvit_base_tf_512.in21k_ft_in1kzmaxvit_large_tf_224.in21kz!maxvit_large_tf_384.in21k_ft_in1kz!maxvit_large_tf_512.in21k_ft_in1k)rrrrzmaxvit_xlarge_tf_224.in21kz"maxvit_xlarge_tf_384.in21k_ft_in1kz"maxvit_xlarge_tf_512.in21k_ft_in1kc [SSU0UD6$)z)CoatNet Pico model with RW configuration.r)coatnet_pico_rw_224rrrs r^rrJ   RZ R6 RRrac [SSU0UD6$)z)CoatNet Nano model with RW configuration.r)coatnet_nano_rw_224rrs r^rrP rrac [SSU0UD6$)z&CoatNet-0 model with RW configuration.r)coatnet_0_rw_224rrs r^rrV   O* O OOrac [SSU0UD6$)z&CoatNet-1 model with RW configuration.r)coatnet_1_rw_224rrs r^rr\ rrac [SSU0UD6$)z&CoatNet-2 model with RW configuration.r)coatnet_2_rw_224rrs r^rrb rrac [SSU0UD6$)z&CoatNet-3 model with RW configuration.r)coatnet_3_rw_224rrs r^rrh rrac [SSU0UD6$)z4CoatNet-0 model with BatchNorm and RW configuration.r)coatnet_bn_0_rw_224rrs r^rrn rrac [SSU0UD6$)z.CoatNet Nano model with Relative Position MLP.r)coatnet_rmlp_nano_rw_224rrs r^rrt   W* WPV WWrac [SSU0UD6$)z+CoatNet-0 model with Relative Position MLP.r)coatnet_rmlp_0_rw_224rrs r^rrz   Tz TV TTrac [SSU0UD6$)z+CoatNet-1 model with Relative Position MLP.r)coatnet_rmlp_1_rw_224rrs r^rr rrac [SSU0UD6$)z.CoatNet-1 model with Relative Position MLP v2.r)coatnet_rmlp_1_rw2_224rrs r^rr s  U Uf UUrac [SSU0UD6$)z+CoatNet-2 model with Relative Position MLP.r)coatnet_rmlp_2_rw_224rrs r^rr rrac [SSU0UD6$)z6CoatNet-2 model with Relative Position MLP at 384x384.r)coatnet_rmlp_2_rw_384rrs r^rr rrac [SSU0UD6$)z+CoatNet-3 model with Relative Position MLP.r)coatnet_rmlp_3_rw_224rrs r^rr rrac [SSU0UD6$)z(CoatNet Nano model with ConvNeXt blocks.r)coatnet_nano_cc_224rrs r^rr rrac [SSU0UD6$)z*CoAtNeXt Nano model with RW configuration.r)coatnext_nano_rw_224rrs r^rr s  Sj SF SSrac [SSU0UD6$)zCoatNet-0 model.r) coatnet_0_224rrs r^rr   Lz LV LLrac [SSU0UD6$)zCoatNet-1 model.r) coatnet_1_224rrs r^rr rrac [SSU0UD6$)zCoatNet-2 model.r) coatnet_2_224rrs r^rr rrac [SSU0UD6$)zCoatNet-3 model.r) coatnet_3_224rrs r^rr rrac [SSU0UD6$)zCoatNet-4 model.r) coatnet_4_224rrs r^rr rrac [SSU0UD6$)zCoatNet-5 model.r) coatnet_5_224rrs r^rr rrac [SSU0UD6$)z(MaxViT Pico model with RW configuration.r)maxvit_pico_rw_256rrs r^rr   QJ Q& QQrac [SSU0UD6$)z(MaxViT Nano model with RW configuration.r)maxvit_nano_rw_256rrs r^r r  rrac [SSU0UD6$)z(MaxViT Tiny model with RW configuration.r)maxvit_tiny_rw_224rrs r^r r  rrac [SSU0UD6$)z3MaxViT Tiny model with RW configuration at 256x256.r)maxvit_tiny_rw_256rrs r^rr rrac [SSU0UD6$)z3MaxViT Relative Position MLP Pico RW 256x256 model.r)maxvit_rmlp_pico_rw_256rrs r^rr   V Vv VVrac [SSU0UD6$)z3MaxViT Relative Position MLP Nano RW 256x256 model.r)maxvit_rmlp_nano_rw_256rrs r^rr rrac [SSU0UD6$)z3MaxViT Relative Position MLP Tiny RW 256x256 model.r)maxvit_rmlp_tiny_rw_256rrs r^rr rrac [SSU0UD6$)z4MaxViT Relative Position MLP Small RW 224x224 model.r)maxvit_rmlp_small_rw_224rrs r^rr rrac [SSU0UD6$)z9MaxViT Small model with Relative Position MLP at 256x256.r)maxvit_rmlp_small_rw_256rrs r^rr rrac [SSU0UD6$)z-MaxViT Base model with Relative Position MLP.r)maxvit_rmlp_base_rw_224rrs r^rr rrac [SSU0UD6$)z8MaxViT Base model with Relative Position MLP at 384x384.r)maxvit_rmlp_base_rw_384rrs r^rr rrac [SSU0UD6$)z'MaxViT Tiny model with parallel blocks.r)maxvit_tiny_pm_256rrs r^rr rrac [SSU0UD6$)z4MaxxViT Relative Position MLP Nano RW 256x256 model.r)maxxvit_rmlp_nano_rw_256rrs r^r!r! rrac [SSU0UD6$)z.MaxxViT Tiny model with Relative Position MLP.r)maxxvit_rmlp_tiny_rw_256rrs r^r#r# rrac [SSU0UD6$)z/MaxxViT Small model with Relative Position MLP.r)maxxvit_rmlp_small_rw_256rrs r^r%r%" s  X: XQW XXrac [SSU0UD6$)zMaxxViT-V2 Nano model.r)maxxvitv2_nano_rw_256rrs r^r'r'( rrac [SSU0UD6$)z1MaxxViT-V2 Base model with Relative Position MLP.r)maxxvitv2_rmlp_base_rw_224rrs r^r)r).   YJ YRX YYrac [SSU0UD6$)zr>d r1rac [SSU0UD6$)z-MaxViT Base model from TensorFlow at 384x384.r)maxvit_base_tf_384rrrs r^r@r@j r1rac [SSU0UD6$)z-MaxViT Base model from TensorFlow at 512x512.r)maxvit_base_tf_512rrrs r^rBrBp r1rac [SSU0UD6$)z#MaxViT Large model from TensorFlow.r)maxvit_large_tf_224rrrs r^rDrDv r8rac [SSU0UD6$)z.MaxViT Large model from TensorFlow at 384x384.r)maxvit_large_tf_384rrrs r^rFrF| r8rac [SSU0UD6$)z.MaxViT Large model from TensorFlow at 512x512.r)maxvit_large_tf_512rrrs r^rHrH r8rac [SSU0UD6$)z$MaxViT XLarge model from TensorFlow.r)maxvit_xlarge_tf_224rrrs r^rJrJ   gR\ g`f ggrac [SSU0UD6$)z/MaxViT XLarge model from TensorFlow at 384x384.r)maxvit_xlarge_tf_384rrrs r^rMrM rKrac [SSU0UD6$)z/MaxViT XLarge model from TensorFlow at 512x512.r)maxvit_xlarge_tf_512rrrs r^rOrO rKrar-) rrFFFrelur{TrTrVNrHrJ) rxrFFg?r{rTrVNr;NrHrJ) rxrFrTrVrTrVNFrXrrJrm)NFrW)rfr1 collectionsr dataclassesrrr functoolsrtypingrr r r r r rrrr torch.jitr timm.datarr timm.layersrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,_builderr. _featuresr/ _features_fxr0 _manipulater1r2 _registryr3r4__all__r7r6r5rXrrrrkrrr4rgr7r9rXrrortryr|rrrrrrrrrrrrrrr8rirjrdrgrjrnrrrrr default_cfgsrrrrrrrrrrrrrrrrrrrrrrrr r rrrrrrrrrr!r#r%r'r)r,r.r0r3r5r7r:r<r>r@rBrDrFrHrJrMrOrmrar^r_s "H #11III A6++34< N 22 2B !P!P !PH  ! !  !Q"))QhR"))Rj/299/d0bii0s0C002NNb,ryy,,S,$,0&8C=&C&C&l"))l^dBIIdN 49 ELL tCy DQTI Z_ZfZf  elltCyU\\ %,, 49 S  V[VbVb   . U38_ QYZbQc D299DNTTnU\\S ell  DI QUVYQZ _d_k_k  5<<DI%,, u|| S  TRUY [`[g[g  K299K\52995p12991hW299Wt/299/d.%S/Ncsz*dbiidP"!& %#)!*.&3)4'+"6666 6 ! 6  6$(6!$6$'6e_666 #s(^6t !&!'!&3)415'+"---- -  - !$ -$'-eCHo.--e_--- #s(^-b ,"-&3)415#9=!$$$$ $ !$ $ $' $eCHo.$$5%u "556$$$ #s(^$Nc3h"n% !  n %  !  n*%  &+ +n< %   &+ =nP '   & Qnd '   & en| %   &+#0  }nR$ %  !   Snh!%  inz! %   &+  {nR" %   Snf! '   &  gn|! '   &  }nT%5  . Unb %  $ cnz% {nF% GnR' Sn^' _nj' knv( wnF$  - GnT%  - Unb%  - cnp%  - qn@#$  5 ) AnN#%  5 ) On\#%  5 ) ]nj$ %   kn~# %   nT$%  + Und$%  + enr%%  + snB ! %   C nX &'   Y nj ' '   k nB %  )C nT %  )U nf %  )g nx '  )y nJ  '  )K n b T#u||*;%<RYYSWX[]b]i]iXiSj, S x} QU il qx  c # $sCx. %X&#Tb\X&"4 H$X& E!FX& E!X& (*!X&*.t0+X&.-d//X&2-d Hsh/X3X&<"4 H "(< $=X&F' M)GX&N&t|OX&P$T J&KQX&V$T J&KWX&\&t|]X&^$Tb\_X&`#D I%aX&l "mX&r "sX&x&t(yX&~%d'X&Ht|IX&Jt|KX&Lt|MX&Nt|OX&Pt|QX&Rt|SX&X#DRMU[$\YX&Z!$ G F#4[X&b!$ G#HcX&h#D  F%4iX&n#DRMU[$\oX&t&t L F(4uX&|&t L F(4}X&D&t L F(4EX&L' M)MX&V)$  F+4WX&`/1aX&f/ Hsh1XgX&p')qX&|' M F)4}X&D)$2-[a*bEX&F( N F*4GX&R$T F&4SX&X244YX&\24 Hsh4X]X&b,Tb\cX&f*4,gX&pt "(< >qX&vt Hsh XwX&|t Hsh X}X&B "(<!>CX&H Hsh!XIX&N Hsh!XOX&Tt "(< >UX&Zt Hsh X[X&`t Hsh XaX&f "(<!>gX&l Hsh!XmX&r Hsh!XsX&z!{X&@' Hsh)XAX&F' Hsh)XGX&L "MX&R( Hsh*XSX&X( 3(*DYX&^!$#_X&d)$ Hsh+XeX&j)$ Hsh+XkX&X vSDSCSGSS SDSCSGSS PPPPP PPPPP PPPPP PPPPP SDSCSGSS XXXXX UdUcUgUU UdUcUgUU VtVsVwVV UdUcUgUU UdUcUgUU UdUcUgUU SDSCSGSS TTTSTWTT MdMcMgMM MdMcMgMM MdMcMgMM MdMcMgMM MdMcMgMM MdMcMgMM R4R3R7RR R4R3R7RR R4R3R7RR R4R3R7RR WWWWW WWWWW WWWWW XXXXX XXXXX WWWWW WWWWW R4R3R7RR XXXXX XXXXX Y$Y#Y'YY UdUcUgUU Z4Z3Z7ZZ Z4Z3Z7ZZ [D[C[G[[ d4d3d7dd d4d3d7dd d4d3d7dd fDfCfGff fDfCfGff fDfCfGff d4d3d7dd d4d3d7dd d4d3d7dd fDfCfGff fDfCfGff fDfCfGff hThShWhh hThShWhh hThShWhhra