QЦi`}SSKrSSKJr SSKJr SSKJrJrJrJ r J r J r SSK r SSKrSSKJs Jr SSKJrJr SSKJrJrJr SSKJr SSKJrJr SS KJrJr SS K J!r! SS K"J#r#J$r$ SS K%J&r& /S Qr'S\ \(\(4S\(S\(S\(S\ \(\(44 Sjr)S\ \(\(4S\(S\\ \(\(44Sjr*S\(S\(S\R&4Sjr+"SS\RX5r-"SS\RX5r."SS\RX5r/"SS \RX5r0"S!S"\RX5r1"S#S$\RX5r2"S%S&\RX5r3"S'S(\RX5r4"S)S*\RX5r5S[TU]5 U US:g=(d X:gn U (aQ[R"XSSSS9/n US:Xa[R"SUSS9/U -n [R "U 6UlO[R"5Ul[X#-5n [X$-5n U(a[USS9Ul O[R"5Ul [5n U"U5U S '[UU SSS UUSS 9U S '[U U SUSUUU SS 9 U S'[X[RS9U S'[R"XSSS9U S'[R "U 5Ulg)Nr#T)rrbiasr"r*rrrrowmodepre_normr)rrrrJrKinplaceconv_a)rrrrJrKgroupsrTconv_b) activationsqueeze_excitation)rFrGrrNconv_c)super__init__r Conv2d AvgPool2d SequentialprojIdentityintrstochastic_depthrrrSiLUlayers)selfrFrGrHrIrrJrKrL should_projr` mid_channels sqz_channels_layers __class__s r%r\MBConv.__init__Csh  k@[%@ IIkQqW[\]D{ 61MNQUU t,DI DI<9: <78 $34Hu$UD !$&KKMD !-(5 0  -!  1  -!  ):,acahah(i$%II,ghostmmG, r'xclURU5nURURU55nX!-$)z Args: x (Tensor): Input tensor with expected layout of [B, C, H, W]. Returns: Tensor: Output tensor with expected layout of [B, C, H / stride, W / stride]. )r`rcrerfrmress r%forwardMBConv.forwards0iil  ! !$++a. 1wr')rer`rc))__name__ __module__ __qualname____firstlineno____doc__rbfloatrr Moduler\r rq__static_attributes__ __classcell__rks@r%rDrD5s ,'*;-;-;- ;-  ;-  ;-#3 >2;-S"))^,;-$;- ;-;-z  F  r'rDct^\rSrSrSrS\S\S\SS4U4SjjrS\R4S jr S \S\4S jr S r U=r $) $RelativePositionalMultiHeadAttentionzRelative Positional Multi-Head Attention. Args: feat_dim (int): Number of input features. head_dim (int): Number of features per head. max_seq_len (int): Maximum sequence length. feat_dimhead_dim max_seq_lenr NcV>[TU]5 X-S:wa[SUSU35eX-UlX l[ [ R"U55UlX0l [R"XRUR-S-5Ul US-Ul [R"URUR-U5Ul[RR!["R$"SUR-S- SUR-S- -UR4["R&S95UlUR+S [-URUR55 ["RR.R1UR(S S 9 g) Nrz feat_dim: z must be divisible by head_dim: r*gr"r#)dtyperelative_position_index{Gz?std)r[r\ ValueErrorn_headsrrbmathsqrtsizerr Linearto_qkv scale_factormerge parameter Parameterr7emptyfloat32relative_position_bias_tableregister_bufferrBinit trunc_normal_)rfrrrrks r%r\-RelativePositionalMultiHeadAttention.__init__sK   ! #z(3ST\S]^_ _+   +./ &ii,,*F*JK $dNYYt}}t||;XF ,.LL,B,B KK!dii-!+DII 0ABDLLQY^YfYf g- ) 68TUYU^U^`d`i`i8jk  ##D$E$E4#Pr'cURRS5nURURURURS5nUR SSS5R 5nUR S5$)Nr6r"rr#)rviewrrr<r= unsqueeze)rf bias_index relative_biass r%get_relative_positional_biasARelativePositionalMultiHeadAttention.get_relative_positional_biasss1166r: 99*EJJ4K[K[]a]m]moqr %--aA6AAC &&q))r'rmcURup#pEURURpvURU5n[R "USSS9upn U R X#XFU5RSSSSS5n U R X#XFU5RSSSSS5n U R X#XFU5RSSSSS5n XR-n [R"SX5n UR5n [R"X-SS9n [R"S X5nURSSSSS5R X#XE5nURU5nU$) z Args: x (Tensor): Input tensor with expected layout of [B, G, P, D]. Returns: Tensor: Output tensor with expected layout of [B, G, P, D]. r*r6)dimrr#r"z!B G H I D, B G H J D -> B G H I Jz!B G H I J, B G H J D -> B G H I D)shaperrrr7chunkreshaper<reinsumrFsoftmaxr)rfrmBGPDHDHqkvqkvdot_prodpos_biasouts r%rq,RelativePositionalMultiHeadAttention.forwards8WW a dmm2kk!n++c1"-a IIaA" % - -aAq! < IIaA" % - -aAq! < IIaA" % - -aAq! < !! !<< CQJ44699X0b9ll>Lkk!Q1a(00q<jjo r')rrrrrrrr) rtrurvrwrxrbr\r7r rrqr{r|r}s@r%rrs`QQQ Q  Q8*ell* Fr'rcv^\rSrSrSrS\S\SS4U4SjjrS\RS\R4S jr S r U=r $) SwapAxeszPermute the axes of a tensor.abr Nc:>[TU]5 XlX lgN)r[r\rr)rfrrrks r%r\SwapAxes.__init__s r'rmc\[R"XRUR5nU$r)r7swapaxesrrros r%rqSwapAxes.forwardsnnQ/ r')rr) rtrurvrwrxrbr\r7r rqr{r|r}s@r%rrs@'##$ %,,r'rcF^\rSrSrSrS U4SjjrS\S\S\4SjrSr U=r $) WindowPartitionz: Partition the input tensor into non-overlapping windows. r c">[TU]5 grr[r\rfrks r%r\WindowPartition.__init__ r'rmpcURup4pVUnURX4XW-XvU-U5nURSSSSSS5nURX5U-Xg--Xw-U5nU$)z Args: x (Tensor): Input tensor with expected layout of [B, C, H, W]. p (int): Number of partitions. Returns: Tensor: Output tensor with expected layout of [B, H/P, W/P, P*P, C]. rr"rr*r#rrr<)rfrmrrCrWrs r%rqWindowPartition.forwardslWW a  IIaAFAAvq 1 IIaAq!Q ' IIaq&QV,aeQ 7r'r$r N rtrurvrwrxr\r rbrqr{r|r}s@r%rrs,CFr'rc N^\rSrSrSrS U4SjjrS\S\S\S\S\4 S jrS r U=r $) WindowDepartitionzg Departition the input tensor of non-overlapping windows into a feature volume of layout [B, C, H, W]. r c">[TU]5 grrrs r%r\WindowDepartition.__init__rr'rmr h_partitions w_partitionscURupVpxUn X4pURXZXX5nURSSSSSS5nURXXX-X-5nU$)a2 Args: x (Tensor): Input tensor with expected layout of [B, (H/P * W/P), P*P, C]. p (int): Number of partitions. h_partitions (int): Number of vertical partitions. w_partitions (int): Number of horizontal partitions. Returns: Tensor: Output tensor with expected layout of [B, C, H, W]. rrr#r*r"rr) rfrmrrrrrPPrrHPWPs r%rqWindowDepartition.forwards`gg b B IIaRA ) IIaAq!Q ' IIaBFBF +r'r$rrr}s@r%rrs;Cs#RXr'rc^\rSrSrSrS\S\S\S\S\\\4S\S \S \ R4S \S \ R4S \ S \ S\ SS4U4Sjjr S\ S\ 4SjrSrU=r$)PartitionAttentionLayeriav Layer for partitioning the input tensor into non-overlapping windows and applying attention to each window. Args: in_channels (int): Number of input channels. head_dim (int): Dimension of each attention head. partition_size (int): Size of the partitions. partition_type (str): Type of partitioning to use. Can be either "grid" or "window". grid_size (Tuple[int, int]): Size of the grid to partition the input tensor into. mlp_ratio (int): Ratio of the feature size expansion in the MLP layer. activation_layer (Callable[..., nn.Module]): Activation function to use. norm_layer (Callable[..., nn.Module]): Normalization function to use. attention_dropout (float): Dropout probability for the attention layer. mlp_dropout (float): Dropout probability for the MLP layer. p_stochastic_dropout (float): Probability of dropping out a partition. rFrpartition_sizepartition_type grid_size mlp_ratiorJ.rKattention_dropout mlp_dropoutrLr Nc >[T U]5 X-UlX lUSU-UlX@lXPlUS;a [S5eUS:XaX0RsUlUl OURUsUlUl [5Ul [5Ul US:Xa [SS5O[R "5UlUS:Xa [SS5O[R "5Ul[R&"U"U5[)XUS-5[R*"U 55Ul[R&"[R."U5[R0"XU-5U"5[R0"X-U5[R*"U 55Ul[5U S S 9Ulg) Nr)gridwindowz0partition_type must be either 'grid' or 'window'rrr"rPrQ)r[r\rr n_partitionsrrrrgr partition_oprdepartition_oprr rapartition_swapdepartition_swapr_rDropout attn_layer LayerNormr mlp_layerrstochastic_dropout) rfrFrrrrrrJrKrrrLrks r%r\ PartitionAttentionLayer.__init__,su" ".  %aLN:," !3 3OP P X %+->->NDFDF!..NDFDF+-/12@F2Jhr2.PRP[P[P]4Bf4LR 0RTR]R]R_-- { # 1XYHY Z JJ( )   LL % IIk#: ;   IIk-{ ; JJ{ #  #22FU"Sr'rmcURSUR-URSUR-p2[R"URSUR-S:H=(a URSUR-S:HSR URUR55 UR XR5nUR U5nXRURU55-nXRURU55-nURU5nURXRX#5nU$)z Args: x (Tensor): Input tensor with expected layout of [B, C, H, W]. Returns: Tensor: Output tensor with expected layout of [B, C, H, W]. rr#z[Grid size must be divisible by partition size. Got grid size of {} and partition size of {}) rrr7_assertformatrrrrrrr)rfrmghgws r%rqPartitionAttentionLayer.forwardfs"dff,dnnQ.?466.IB NN1  &! + Oq0ADFF0Ja0O i p p     a (    " ''(:; ; ''q(9: :  ! !! $   662 2r')rrrrrrrrrrrrrr)rtrurvrwrxrbstrr rr rzryr\r rqr{r|r}s@r%rrs"8T8T8T  8T  8Tc?8T8T#3 >28TS"))^,8T!8T8T$8T !8TtFr'rc^\rSrSrSrS\S\S\S\S\S\S \R4S \S \R4S \S \S \S\S\S\S\ \\4SS4U4Sjjr S\ S\ 4Sjr SrU=r$) MaxVitLayeria MaxVit layer consisting of a MBConv layer followed by a PartitionAttentionLayer with `window` and a PartitionAttentionLayer with `grid`. Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. expansion_ratio (float): Expansion ratio in the bottleneck. squeeze_ratio (float): Squeeze ratio in the SE Layer. stride (int): Stride of the depthwise convolution. activation_layer (Callable[..., nn.Module]): Activation function. norm_layer (Callable[..., nn.Module]): Normalization function. head_dim (int): Dimension of the attention heads. mlp_ratio (int): Ratio of the MLP layer. mlp_dropout (float): Dropout probability for the MLP layer. attention_dropout (float): Dropout probability for the attention layer. p_stochastic_dropout (float): Probability of stochastic depth. partition_size (int): Size of the partitions. grid_size (Tuple[int, int]): Size of the input feature grid. rFrGrIrHrrK.rJrrrrrLrrr Nc">[TU]5 [5n[UUUUUUUU S9US'[ UUU SUU U[ R U U U S9 US'[ UUU SUU U[ R U U U S9 US'[ R"U5Ulg)N)rFrGrHrIrrJrKrLMBconvr) rFrrrrrrJrKrrrLwindow_attentionrgrid_attention) r[r\rrDrr rr_re)rfrFrGrIrHrrKrJrrrrrLrrrerks r%r\MaxVitLayer.__init__s* )m"#%+'-!!5  x&=$)#-||/#!5 & !"$;$)!-||/#!5 $  mmF+ r'rmc(URU5nU$zu Args: x (Tensor): Input tensor of shape (B, C, H, W). Returns: Tensor: Output tensor of shape (B, C, H, W). re)rfrms r%rqMaxVitLayer.forwards KKNr'r )rtrurvrwrxrbryrr rzr r\r rqr{r|r}s@r%rrs(?,?, ?,  ?,  ?,?,S"))^,?,#3 >2?,?,?,?,!?, $!?,$%?,&c?'?,( )?,BFr'rc^\rSrSrSrS\S\S\S\S\S\R4S \S\R4S \S \S \S \S\S\ \\4S\S\ \SS4U4Sjjr S\ S\ 4SjrSrU=r$) MaxVitBlockia A MaxVit block consisting of `n_layers` MaxVit layers. Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. expansion_ratio (float): Expansion ratio in the bottleneck. squeeze_ratio (float): Squeeze ratio in the SE Layer. activation_layer (Callable[..., nn.Module]): Activation function. norm_layer (Callable[..., nn.Module]): Normalization function. head_dim (int): Dimension of the attention heads. mlp_ratio (int): Ratio of the MLP layer. mlp_dropout (float): Dropout probability for the MLP layer. attention_dropout (float): Dropout probability for the attention layer. p_stochastic_dropout (float): Probability of stochastic depth. partition_size (int): Size of the partitions. input_grid_size (Tuple[int, int]): Size of the input feature grid. n_layers (int): Number of layers in the block. p_stochastic (List[float]): List of probabilities for stochastic depth for each layer. rFrGrIrHrK.rJrrrrrinput_grid_sizen_layers p_stochasticr Ncp>[TU]5 [U5U :Xd[SU SUS35e[R "5Ul[U SSSS9Ul[U5HLunnUS:XaSOSnU=R [US:XaUOUUUUUUUUUU U U URUS 9/- slMN g) Nz'p_stochastic must have length n_layers=z, got p_stochastic=.r*r"r#rOr)rFrGrIrHrrKrJrrrrrrrL) r[r\lenrr ModuleListrer&r enumerater)rfrFrGrIrHrKrJrrrrrrrridxrrrks r%r\MaxVitBlock.__init__s, < H,FxjPcdpcqqrst tmmo /QWXbcd -FC(QF KK/2ax \!-"/$3!)%5%' +&7#1"nn)* K.r'rmc<URH nU"U5nM U$r r )rfrmlayers r%rqMaxVitBlock.forward,s [[EaA!r')rre)rtrurvrwrxrbryrr rzr rr\r rqr{r|r}s@r%rrs*11 1  1  1S"))^,1#3 >21111!1 !1"sCx#1&'1(5k)1* +1f  F  r'rc!^\rSrSrSrS\R SSSSSS4S\\\4S \S \S \ \S \ \S \S\ S\ \ S\R4S\ S\R4S\ S\ S\S\ S\ S\SS4 U4SjjjrS\S\4SjrSrSrU=r$)ri8a1 Implements MaxVit Transformer from the `MaxViT: Multi-Axis Vision Transformer `_ paper. Args: input_size (Tuple[int, int]): Size of the input image. stem_channels (int): Number of channels in the stem. partition_size (int): Size of the partitions. block_channels (List[int]): Number of channels in each block. block_layers (List[int]): Number of layers in each block. stochastic_depth_prob (float): Probability of stochastic depth. Expands to a list of probabilities for each layer that scales linearly to the specified value. squeeze_ratio (float): Squeeze ratio in the SE Layer. Default: 0.25. expansion_ratio (float): Expansion ratio in the MBConv bottleneck. Default: 4. norm_layer (Callable[..., nn.Module]): Normalization function. Default: None (setting to None will produce a `BatchNorm2d(eps=1e-3, momentum=0.01)`). activation_layer (Callable[..., nn.Module]): Activation function Default: nn.GELU. head_dim (int): Dimension of the attention heads. mlp_ratio (int): Expansion ratio of the MLP layer. Default: 4. mlp_dropout (float): Dropout probability for the MLP layer. Default: 0.0. attention_dropout (float): Dropout probability for the attention layer. Default: 0.0. num_classes (int): Number of classes. Default: 1000. Ng?rrsir stem_channelsrblock_channels block_layersrstochastic_depth_probrK.rJrIrHrrr num_classesr c>[TU]5 [U5 SnUc[[R SSS9n[ U[U55n[U5H6unnUSU-S:wdUSU-S:wdM [SUSUS US US 3 5e [R"[UUSS UU S SS9[X"SSSSSS95Ul [USS SS9nX0l[R"5UlU/USS-nUn["R$"SU['U55R)5nSn[+UUU5HZunnnUR R-[/UUU U UU UU U UUUUUUUU-S95 UR SR0nUU- nM\ [R"[R2"S5[R4"5[R6"US5[R8"USUS5[R:"5[R8"USUS S95UlUR?5 g)Nr*gMbP?g{Gz?)epsmomentumrr#z Input size z of block z$ is not divisible by partition size zx. Consider changing the partition size or the input size. Current configuration yields the following block input sizes: rr"F)rrKrJrNrTT)rrKrJrNrOr6)rFrGrIrHrKrJrrrrrrrr)rN) r[r\rrr BatchNorm2dr0rrrr_rstemr&rrblocksnplinspacer>tolistzipr,rrAdaptiveAvgPool2dFlattenrrTanh classifier _init_weights)rfrrrrrrr rKrJrIrHrrrr!input_channelsblock_input_sizesrblock_input_sizerFrGrp_idx in_channel out_channel num_layersrks r%r\MaxVit.__init__Ms: D!   TDIJ 5Z^ATU%./@%A !C!"^3q857WW#!" #&7t#s(Ot  ttS t3itt %t"Xc299n56#t$#3 >2%t()t*+t./t01t2!3t67t8 9ttlF + +r'rrrrr rrweightsprogresskwargsc dUbe[US[URS55 URSSURSS:Xde[USURS5 URSS5n [ S UUUUUUU S.UD6n UbU R UR US S 95 U $) Nr! categoriesmin_sizerr#rrM)rrrr rrrT)rG check_hashr$)rrmetapoprload_state_dictget_state_dict) rrrr rrrFrGrHrmodels r%_maxvitrTs$fmSl9S5TU||J'*gll:.Fq.IIIIflGLL4LML*5J  #%!3%   E g44hSW4XY Lr'c f\rSrSr\"S\"\SS\RS9\ SSSSS S S .0S S SS.S9r \ r Sr g)riz9https://download.pytorch.org/models/maxvit_t-bc5ab103.pthrM) crop_size resize_size interpolationirLzLhttps://github.com/pytorch/vision/tree/main/references/classification#maxvitz ImageNet-1KgT@g|?5.X@)zacc@1zacc@5gZd;@gK7]@zThese weights reproduce closely the results of the paper using a similar training recipe. They were trained with a BatchNorm2D momentum of 0.99 instead of the more correct 0.01.)rJ num_paramsrKrecipe_metrics_ops _file_size_docs)url transformsrOr$N) rtrurvrwrrrrBICUBICr IMAGENET1K_V1DEFAULTr{r$r'r%rrsb G 3CO`OhOh /""d##  !g M.Gr'r pretrained)rFT)rFrGc \[RU5n[SS/SQ/SQSSSUUS.UD6$) aF Constructs a maxvit_t architecture from `MaxViT: Multi-Axis Vision Transformer `_. Args: weights (:class:`~torchvision.models.MaxVit_T_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.MaxVit_T_Weights` below for more details, and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.maxvit.MaxVit`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.MaxVit_T_Weights :members: @)rfi)r"r"rr" g?)rrrrr rrFrGr$)rverifyrT)rFrGrHs r%rrsH.%%g.G  *!!    r')NF)r{sX #AA HH9TH<R1 uS#XSRU`chmnqsvnvhwsCxCDQVWZ\_W_Q`La##S#U\\#TRYYTnF299FR ryy bii4