
    RЦi                     &   S r SSKrSSKJrJrJr  SSKJr  SSKJ	r	J
r
JrJrJrJrJrJrJr  SSKrSSKJr  SSKJrJrJrJr  SSKJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+  SS	K,J-r-  SS
K.J/r/  SSK0J1r1J2r2  SSK3J4r4J5r5  / SQr6\ " S S5      5       r7\ " S S5      5       r8GSS\	\9S4   S\	\:S4   S\9S\	\7S4   4S jjr;    GSS\	\9S4   S\	\:S4   S\	\9S4   S\9S\
\
\7      4
S jjr<  GSS\	\=\=4   S\9S\\9\
\9   4   S\>S\	\7S4   4
S jjr?S \\7\\7   4   S\
\7   4S! jr@S"\\9   S#\9S\94S$ jrA\ " S% S&5      5       rB " S' S(\R                  5      rDS)\=S*\9S+\9S,\9S-\	\9\94   S.\BS\\R                     4S/ jrE " S0 S1\R                  5      rF " S2 S3\R                  5      rG " S4 S5\R                  5      rH " S6 S7\R                  5      rI " S8 S9\R                  5      rJ " S: S;\R                  5      rK " S< S=\R                  5      rL\M" \F\G\H\I\J\K\LS>9rNS?\=S@\R                  4SA jrOSB\\=\R                  4   4SC jrP " SD SE\R                  5      rR      GSS*\9S+\9SG\=SH\=SI\=S.\B4SJ jjrSGSSL jrTSM rUSN\\=\4   SO\7SP\84SQ jrV   GSSU\:SV\9SW\9S\
\\      4SX jjrWSRSSSS\VSS4SY\8SZ\:S[\9S\\\=\4   S]\:S^\9S_\\9   S.\\B   S`\\   4Sa jjrXGSSY\8Sc\>4Sd jjrY " Se Sf\R                  5      rZGSSg\R                  Sh\=Si\>SS4Sj jjr[\M" S0 Sk\8" \7" SlSSmSKSSnSo9\7" SlSKSpSKSSnSo9\7" SqSrSsSKSStSo9\7" SqSuSsSKSSvSo9\7" SqSTSsSSSvSo94SwSSxSy9_Sz\8" \7" SlSSmSKSSnSo9\7" SlSKSpSKSSnSo9\7" SqSrSsSKSStSo9\7" SqSTSsSKSSvSo9\7" SqSSsSSSvSo94SwSSxSy9_S{\8" \7" SlSS|SKSSnSo9\7" SlSSS|SKSSnSo9\7" SqS}S~SKSStSo9\7" SqSKSSKSSvSo9\7" SqSSSSSvSo94SSSSy9_S\8" \;" SSS9SS|S9_S\8" \;" SSS9SSS9_S\8" \;" SSS9SSS9_S\8" \;" SS9SSS9_S\8" \;" SS9SSS9_S\8" \;" SSTS9SSS9_S\8" \;" SS9SSS9_S\8" \;" SSTS9SSS9_S\8" \;" SS9SSS9_S\8" \;" SSTS9SSS9_S\8" \;" SSS9SSS\M" SSS9S9_S\8" \7" SqSKSSSwStSo9\7" SqSTSSKSwStSo9\7" SqSrSSKSwStSo9\7" SqSTSSKSSnSo94SmSSSSS9_S\8" \7" SSSSSSn\M" 5       S9\7" SqSTSSKSwStSo9\7" SqSrSSKSwStSo9\7" SqSTSSKSSnSo94SmSSSS\M" SbS9S9_S\8" \7" SqSKSSSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo94SSSSS9_S\8" \7" SqSKSSSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo94SSSSSS9_S\8" \7" SqSKSSSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo94SSSSSS9_S\8" \7" SqSKSSSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo94SSSSSS9_S\8" \7" SqSKSSSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo9\7" SqSKSSKSwStSo94SSSSS\M" SS9S9_S\8" \7" SqSKSSSStSo9\7" SqSSSSKSStSo9\7" SqSSSSKSStSo9\7" SqSKSSKSStSo94SSSFSSS9_S\8" \7" SqSKSSSStSo9\7" SqSSSSKSStSo9\7" SqSSSSKSStSo9\7" SqSKSSKSStSo94SSSFSSS9_S\8" \7" SqSKSSSStSo9\7" SqSSSSKSStSo9\7" SqSSSSKSStSo9\7" SqSKSSKSStSo94SSSFSSSS9_S\8" \7" SqSKSSSStSo9\7" SqSSSSKSStSo9\7" SqSSSSKSStSo9\7" SqSKSSKSStSo94SSSFSSSS9_S\8" \7" SqSKSSSStSo9\7" SqSSSSKSStSo9\7" SqSSSSKSStSo9\7" SqSKSSKSStSo94SSSFSSSS9_S\8" \7" SqSSSSStS9\7" SqSTSSKStS9\7" SqSrSSKStS9\7" SqSSSSKStS94SSSFSS9_S\8" \7" SqSSSSSwStSo9\7" SqSTSSKSwStSo9\7" SqSrSSKSwStSo9\7" SqSSSSKSwStSo94SSSSSS9_S\8" \7" SqSKS|SKSSSSo9\7" SqSrSSKSSSSo9\7" SqSSpSKSSSSo9\7" SqSKSSKSSSSo94SwSFSFSSS\M" StS9\M" SbSbS9S9	_S\8" \7" SqSKS|SKSSTSo9\7" SqSrSSKSSTSo9\7" SqSSpSKSSTSo9\7" SqSKSSKSSTSo94SwSFSFSSS\M" StS9\M" SbSbS9S9	_S\8" \7" SqSSSSSwSTSo9\7" SqSrSmSKSwSTSo9\7" SqSSSKSwSTSo9\7" SqSSS~SKSwSTSo94SSSFSFSSS\M" StS9\M" SbSbS9S9
_S\8" \7" SqSSSSSSTSo9\7" SqSrSmSKSSTSo9\7" SqSSSKSSTSo9\7" SqSSS~SKSSTSo94SSSFSFSSS\M" StS9\M" SbSbS9S9
_S\8" \7" SqSSSSSSTSo9\7" SqSSpSKSSTSo9\7" SqSS~SKSSTSo9\7" SqSSSSKSSTSo94SSSFSFSSS\M" StS9\M" SbSbS9S9
_S\8" \7" SqSKS|SKSSSSo9\7" SqSrSSKSSSSo9\7" SqSSpSKSSSSo9\7" SqSKSSKSSSSo94SwSFSFSS\" \ SS9S\M" StS9\M" SbSbS9S9
_S\8" \7" SqSKS|SKSSTSo9\7" SqSrSSKSSTSo9\7" SqSSpSKSSTSo9\7" SqSKSSKSSTSo94SwSFSFSS\" \ SS9S\M" StS9\M" SbSbS9S9
_S\8" \7" SqSSSSSSTSo9\7" SqSrSmSKSSTSo9\7" SqSSSKSSTSo9\7" SqSSS~SKSSTSo94SSSFSFSS\" \ SS9S\M" StS9\M" SbSbS9S9_S\8" \<" SSTS9SS|S9_S\8" \<" SS9SSS9_S\8" \<" SS9SSS9_S\8" \<" SS9SSS9_S\8" \<" SSS9SSS9_S\8" \7" SqSSSSStS9\7" SqSTSSKStS9\7" SqSrSSKStS9\7" SqSSSSKStS94SSFSSSSS9_S\8" \7" SqSSSSStS9\7" SqSTSSKStS9\7" SqSSSKStS9\7" SqSSSSKStS94SSFSSSSS9_S\8" \7" SqSTSSStS9\7" SqSrSSKStS9\7" SqSSSKStS9\7" SqSrSSKStS94SSSFSSSSS9_S\8" \7" SqSrSSStS9\7" SqSSSKStS9\7" SqSSSKStS9\7" SqSSSKStS94SSSFSSSSS9_S\8" \7" SqSSSSStS9\7" SqSSSKStS9\7" SqSSSKStS9\7" SqSSSKStS94SSSFSSSSS9_S\8" \7" SqSSSSStS9\7" SqSTSSKStS9\7" SqSrSSKStS9\7" SqSSSSKStS94SSFSSSSGS GS9_GS\8" \7" SSSwSKSGSSo9\7" GSSSSKSGSSo9\7" SlSSmSKSwStSo9\7" SqSSSKSStSo94GSSSFGSS\M" StS9GS9_6r\GS H  r]\" \\\]   GS	GS
9\\\]GS-   '   M      GSGS\\=\R                  4   GS\ZGS\=S\\=\R                  4   4GS jjr_GS\\=\R                  4   GS\Z4GS jr`GSGS\=GS\>S\Z4GS jjraGSGS\=S\\=\4   4GS jjrbGSGS\=S\\=\4   4GS jjrc\4" 0 GS\b" GSGS9_GS\b" GSGS9_GS\b" GSGSGSGS9_GS\b" GSGS GS!GS"9_GS#\b" GSGS GS!GS"9_GS$\b" GSGS GS!GS"9_GS%\b" GSGS GS!GS"9_GS&\b" GSGS GS!GS"9_GS'\b" GSGS GS!GS"9_GS(\b" GSGS GS!GS"9_GS)\b" GSGS GS!GS"9_GS*\b" GSGS GS!GS"9_GS+\b" GSGS GS!GS"9_GS,\b" GSGS GS!GS-GS.SnGS/9_GS0\b" GSGS1GS2GSGSGS3SnGS49_GS5\c" GSGS6GS3SnGS79_GS8\c" GSGS9GS3SnGS79_0 GS:\c" GSGS;GS3SnGS79_GS<\c" GSGS=GS3SnGS79_GS>\c" GSGS?GS3SnGS79_GS@\c" GSGSAGSGSB9_GSC\c" GSGSDGS3SnGS79_GSE\c" GSGSFGS3SnGS79_GSG\c" GSGSHGS3SnGS79_GSI\c" GSGSJGS3SnGS79_GSK\c" GSGSLGS3SnGS79_GSM\c" GSGSNGS3SnGS79_GSO\c" GSGSPGS3SnGS79_GSQ\c" GSGSRGSSGSTGSTGSUGSVGSWGS3SnGSX9
_GSY\c" GSGSZGSSGSTGSTGSWGS-SnGS[9_GS\\c" GSGS]GSTGSTGS^GS-GS_9_GS`\c" GSGSaGSTGSTGSWGS-SnGSb9_GSc\c" GSGSdGSTGSTGSWGS-SnGSb9_GSe\c" GSSGSTGSTGSUGSVGS^GS3GSf9_E0 GSg\c" GSGShGSSGSTGSTGS^GS-GSi9_GSj\c" GSGSkGSTGSTGS^GS-SnGSb9_GSl\b" GSGSmGSnGSoGSp9_GSq\b" GSGSrGSnGSoGSp9_GSs\b" GSGSrGSnGSoGSp9_GSt\b" GSGSrGSnGSoGSp9_GSu\b" GSGSrGSnGSoGSp9_GSv\c" GSS\\SbGSUGSVGSwGS!GSx9	_GSy\c" GSS\\SbGSUGSVGSwGS!GSx9	_GSz\c" GSSs\\SbGS3GS{GSwGS!GSx9	_GS|\c" GSGS}\\SbGS~GSGSwGS!GSx9	_GS\c" GSS\\SbGSGSGSwGS!GSx9	_GS\c" GSS\\SbGSUGSVGSwGS!GSx9	_GS\c" GSS\\SbGSUGSVGSwGS!GSx9	_GS\c" GSS\\SbGSUGSVGSwGS!GSx9	_GS\c" GSS\\GSUGSVGS!GS9_GS\c" GSS\\GSUGSVGS!GS9_E\c" GSS\\GS3GS{GS!GS9\c" GSS\\GS~GSGS!GS9\c" GSS\\GSGSGS!GS9\c" GSS\\GSUGSVGS!GS9\c" GSS\\GSUGSVGS!GS9\c" GSS\\GSUGSVGS!GS9\c" GSGSGS9\c" GSGSSGSGS^GSGS9GS.E5      rd\5GSS\Z4GS jj5       re\5GSS\Z4GS jj5       rf\5GSS\Z4GS jj5       rg\5GSS\Z4GS jj5       rh\5GSS\Z4GS jj5       ri\5GSS\Z4GS jj5       rj\5GSS\Z4GS jj5       rk\5GSS\Z4GS jj5       rl\5GSS\Z4GS jj5       rm\5GSS\Z4GS jj5       rn\5GSS\Z4GS jj5       ro\5GSS\Z4GS jj5       rp\5GSS\Z4GS jj5       rq\5GSS\Z4GS jj5       rr\5GSS\Z4GS jj5       rs\5GSS\Z4GS jj5       rt\5GSS\Z4GS jj5       ru\5GSS\Z4GS jj5       rv\5GSS\Z4GS jj5       rw\5GSS\Z4GS jj5       rx\5GSS\Z4GS jj5       ry\5GSS\Z4GS jj5       rz\5GSS\Z4GS jj5       r{\5GSS\Z4GS jj5       r|\5GSS\Z4GS jj5       r}\5GSS\Z4GS jj5       r~\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       r\5GSS\Z4GS jj5       rg(  a  Bring-Your-Own-Blocks Network

A flexible network w/ dataclass based config for stacking those NN blocks.

This model is currently used to implement the following networks:

GPU Efficient (ResNets) - gernet_l/m/s (original versions called genet, but this was already used (by SENet author)).
Paper: `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
Code and weights: https://github.com/idstcv/GPU-Efficient-Networks, licensed Apache 2.0

RepVGG - repvgg_*
Paper: `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
Code and weights: https://github.com/DingXiaoH/RepVGG, licensed MIT

MobileOne - mobileone_*
Paper: `MobileOne: An Improved One millisecond Mobile Backbone` - https://arxiv.org/abs/2206.04040
Code and weights: https://github.com/apple/ml-mobileone, licensed MIT

In all cases the models have been modified to fit within the design of ByobNet. I've remapped
the original weights and verified accuracies.

For GPU Efficient nets, I used the original names for the blocks since they were for the most part
the same as original residual blocks in ResNe(X)t, DarkNet, and other existing models. Note also some
changes introduced in RegNet were also present in the stem and bottleneck blocks for this model.

A significant number of different network archs can be implemented here, including variants of the
above nets that include attention.

Hacked together by / copyright Ross Wightman, 2021.
    N)	dataclassfieldreplace)partial)	TupleListDictOptionalUnionAnyCallableSequenceType)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDOPENAI_CLIP_MEANOPENAI_CLIP_STD)ClassifierHeadNormMlpClassifierHeadConvNormActBatchNormAct2dDropBlock2dEvoNorm2dS0aAttentionPool2dRotAttentionPool2dDropPathcalculate_drop_path_ratesAvgPool2dSamecreate_conv2dget_act_layerget_norm_act_layerget_attnmake_divisible	to_2tuple   )build_model_with_cfg)feature_take_indices)named_applycheckpoint_seq)generate_default_cfgsregister_model)ByobNetByoModelCfgByoBlockCfgcreate_byob_stemcreate_blockc                      \ rS rSr% Sr\\\R                  4   \	S'   \
\	S'   \
\	S'   Sr\
\	S'   Sr\\\
\4      \	S	'   S
r\\	S'   Sr\\   \	S'   Sr\\\\4      \	S'   Sr\\   \	S'   Sr\\\\4      \	S'   Sr\\\\4      \	S'   Srg)r.   C   zmBlock configuration for Bring-Your-Own-Blocks.

Defines configuration for a single block or stage of blocks.
typedc   sNgs      ?br
attn_layerattn_kwargsself_attn_layerself_attn_kwargsblock_kwargs )__name__
__module____qualname____firstlineno____doc__r   strnnModule__annotations__intr7   r8   r
   r   r:   floatr;   r<   r	   r   r=   r>   r?   __static_attributes__r@       R/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/byobnet.pyr.   r.   C   s     RYY

F
FAsJ)-BsH}%&-BN !%J$,0K$sCx.)0%)OXc])15htCH~.5-1L(4S>*1rM   r.   c                      \ rS rSr% Sr\\\\\S4   4   S4   \S'   Sr	\
\S'   Sr\
\S'   S	r\\
   \S
'   Sr\\\\   \\S4   4   \S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\
\S'   Sr\
\S'   Sr\
\S'   Sr\\   \S'   Sr\
\S'   Sr\\
   \S'   \" S  S!9r\\S"'   Sr \\
   \S#'   \" S$ S!9r!\\S%'   \" S& S!9r"\#\
\$4   \S''   S(r%g))r-   X   zdModel configuration for Bring-Your-Own-Blocks network.

Defines overall architecture configuration.
.blocksconv1x1
downsample3x3	stem_typemaxpool	stem_pool    stem_chsr9   width_factorr   num_featuresTzero_init_lastFfixed_input_sizerelu	act_layer	batchnorm
norm_layer aa_layerNhead_hidden_size
classifier	head_typer;   c                      [        5       $ Ndictr@   rM   rN   <lambda>ByoModelCfg.<lambda>t   s    dfrM   )default_factoryr<   r=   c                      [        5       $ rh   ri   r@   rM   rN   rk   rl   v   s    46rM   r>   c                      [        5       $ rh   ri   r@   rM   rN   rk   rl   w   s    rM   r?   r@   )&rA   rB   rC   rD   rE   r   r   r.   rI   rS   rF   rU   rW   r
   rY   rJ   r   rZ   rK   r[   r\   boolr]   r_   ra   rc   rd   rf   r;   r   r<   rj   r=   r>   r?   r	   r   rL   r@   rM   rN   r-   r-   X   s)    %U;+;%<<=sBCCJIs(Ix}(79HeCcE#s(O349L%L#ND"d" Is!J!Hc '+hsm*!Is! !%J$n=K=%)OXc])">BdB#(#HL$sCx.HrM   r-   r4   .wfgroupsreturnc                    ^ SnSnTS:  a  U4S jn[        [        XU5       V VVs/ s H  u  pn[        SXU-  US9PM     snnn 5      nU$ s  snnn f )zCreate RepVGG block configuration.

Args:
    d: Depth (number of blocks) per stage.
    wf: Width factor per stage.
    groups: Number of groups for grouped convolution.

Returns:
    Tuple of block configurations.
@            r   c                 *   > US-   S-  S:X  a  U T-  $ S$ )Nr%   r6   r   r@   )chsidxrr   s     rN   rk   _rep_vgg_bcfg.<locals>.<lambda>   s     a1}7IcVm&Pq&PrM   rep)r3   r4   r5   r8   )tuplezipr.   )r4   rq   rr   r5   
group_sizebcfgs     `   rN   _rep_vgg_bcfgr   z   s]     	AJzP
X[\]bdXefXeHART+5ARJGXefgDK gs   Ar@   	se_blocksnum_conv_branchesc                 j   Sn[        SUS   US   -  5      nU=(       d    S[        U 5      -  n/ n[        XX5       Hr  u  ppx/ n	[        U 5       HV  n
XG-  n[	        US9n0 nXU-
  :  a  SUS'   U	[        SSS	US	US
.UD6/-  n	U	[        SSS	US[	        SSS	0UD6S
.UD6/-  n	UnMX     Xi/-  nMt     U$ )a  Create MobileOne block configuration.

Args:
    d: Depth (number of blocks) per stage.
    wf: Width factor per stage.
    se_blocks: Number of SE blocks per stage.
    num_conv_branches: Number of conv branches.

Returns:
    List of block configurations per stage.
ru   rv   r   )r   )r   ser;   oner%   )r3   r4   r5   r8   r?   kernel_sizer@   )minlenr   rangerj   r.   )r4   rq   r   r   r5   prev_cr   wr   scfgiout_cbkaks                 rN   _mobileone_bcfgr      s   " 	AQqTBqE\"F*TCF]ID1/aqAEE(9:BBF{#'< [XeqFqrXUWXYYD[ ^a5QT=Va=VSU=V^Z\^ _ _DF  	 0 KrM   typeseveryfirstc                 @   [        U 5      S:X  d   e[        U[        5      (       a.  [        [	        U(       a  SOUXS-   5      5      nU(       d  US-
  /n[        U5        / n[	        U5       H$  nXb;   a  U S   OU S   nU[        SUSS.UD6/-  nM&     [        U5      $ )a8  Interleave 2 block types in stack.

Args:
    types: Two block type names to interleave.
    d: Total depth of blocks.
    every: Interval for alternating blocks.
    first: Whether to start with alternate block.
    **kwargs: Additional block arguments.

Returns:
    Tuple of interleaved block configurations.
r6   r   r%   )r3   r4   r@   )r   
isinstancerJ   listr   setr.   r   )r   r4   r   r   kwargsrQ   r   
block_types           rN   interleave_blocksr      s    & u:??%U15!QY?@UGEJF1X!"U1Xq
;>J!>v>??  =rM   stage_blocks_cfgc           
          [        U [        5      (       d  U 4n / n[        U 5       H6  u  p#U[        UR                  5       Vs/ s H  n[        USS9PM     sn-  nM8     U$ s  snf )zExpand block config into individual block instances.

Args:
    stage_blocks_cfg: Block configuration(s) for a stage.

Returns:
    List of individual block configurations.
r%   )r4   )r   r   	enumerater   r4   r   )r   
block_cfgsr   cfg_s        rN   expand_blocks_cfgr      sf     &11,.J,-%,?,Qwsa(,??
 . @s   A"r   channelsc                 .    U (       d  gX-  S:X  d   eX-  $ )zCalculate number of groups for grouped convolution.

Args:
    group_size: Size of each group (1 for depthwise).
    channels: Number of channels.

Returns:
    Number of groups.
r%   r   r@   )r   r   s     rN   
num_groupsr      s&      $)))%%rM   c                      \ rS rSr% Sr\r\\R                     \
S'   \r\\R                     \
S'   \R                  r\\R                     \
S'   Sr\\\R                        \
S'   Sr\\\R                        \
S'   S	rg)
LayerFn   z&Container for layer factory functions.conv_norm_actnorm_actactNattn	self_attnr@   )rA   rB   rC   rD   rE   r   r   r   rG   rH   rI   r   r   ReLUr   r   r
   r   rL   r@   rM   rN   r   r      sm    0%0M4		?0 .Hd299o.77Cbii"&*D(4		?
#*+/IxRYY(/rM   r   c                      ^  \ rS rSrSr      SS\S\S\S\S\S\\   4U 4S	 jjjr	S
\
R                  S\
R                  4S jrSrU =r$ )DownsampleAvgrx   zSAverage pool downsampling module.

AvgPool Downsampling as in 'D' ResNet variants.
in_chsout_chsstridedilation	apply_actlayersc	                 H  > XxS.n	[         TU ]  5         U=(       d
    [        5       nUS:X  a  UOSn
US:  d  US:  a1  U
S:X  a  US:  a  [        O[        R
                  nU" SU
SSS9U l        O[        R                  " 5       U l        UR                  " XS4SU0U	D6U l	        g)	a  Initialize DownsampleAvg.

Args:
    in_chs: Number of input channels.
    out_chs: Number of output channels.
    stride: Stride for downsampling.
    dilation: Dilation rate.
    apply_act: Whether to apply activation.
    layers: Layer factory functions.
devicedtyper%   r6   TF)	ceil_modecount_include_padr   N)
super__init__r   r   rG   	AvgPool2dpoolIdentityr   conv)selfr   r   r   r   r   r   r   r   dd
avg_strideavg_pool_fn	__class__s               rN   r   DownsampleAvg.__init__  s    * /$79'1}V!
A:A+5?x!|-QSQ]Q]K#AzTUZ[DIDI((!WyWTVW	rM   xrs   c                 B    U R                  U R                  U5      5      $ )zGForward pass.

Args:
    x: Input tensor.

Returns:
    Output tensor.
r   r   r   r   s     rN   forwardDownsampleAvg.forward&  s     yy1&&rM   r   )r%   r%   FNNN)rA   rB   rC   rD   rE   rJ   rp   r
   r   r   torchTensorr   rL   __classcell__r   s   @rN   r   r      s     #(,XX X 	X
 X X W%X X@	' 	'%,, 	' 	'rM   r   downsample_typer   r   r   r   r   c                     U S;   d   eX:w  d  US:w  d  US   US   :w  a8  U (       d  gU S:X  a  [        X4X4S   S.UD6$ UR                  " X4SX4S   S.UD6$ [        R                  " 5       $ )ai  Create shortcut connection for residual blocks.

Args:
    downsample_type: Type of downsampling ('avg', 'conv1x1', or '').
    in_chs: Input channels.
    out_chs: Output channels.
    stride: Stride for downsampling.
    dilation: Dilation rates.
    layers: Layer factory functions.
    **kwargs: Additional arguments.

Returns:
    Shortcut module or None.
)avgrR   rb   r%   r   Nr   r   r   )r   r   r   )r   r   rG   r   )r   r   r   r   r   r   r   s          rN   create_shortcutr   2  s    . 4444FaK8A;(1++E% `ST+`Y_``''vQvij`kvouvv{{}rM   c                      ^  \ rS rSrSr             SS\S\S\S\S\\\4   S\\   S	\S
\	S\
S\
S\S\S\4U 4S jjjrSS\
4S jjrS rSrU =r$ )
BasicBlockiU  z#ResNet Basic Block - kxk + kxk
    r   r   r   r   r   r   bottle_ratiorS   	attn_last
linear_outr   
drop_blockdrop_path_ratec                   > XS.n[         TU ]  5         U=(       d
    [        5       n[        X'-  5      n[	        UU5      n[        UUU4UUSUS.UD6U l        UR                  " UUU4XES   S.UD6U l        U	(       d  UR                  c  [        R                  " 5       OUR                  U5      U l	        UR                  " UUU4US   UUSS.UD6U l        U	(       a  UR                  c  [        R                  " 5       OUR                  " U40 UD6U l        US:  a  [        U5      O[        R                  " 5       U l        U
(       a  [        R                  " 5       U l        g UR!                  S	S
9U l        g )Nr   Fr   r   r   r   r   r   r%   )r   rr   
drop_layerr           Tinplace)r   r   r   r#   r   r   shortcutr   	conv1_kxkr   rG   r   	conv2_kxkr   r   	drop_pathr   )r   r   r   r   r   r   r   r   rS   r   r   r   r   r   r   r   r   mid_chsrr   r   s                      rN   r   BasicBlock.__init__Y  sc   $ /$79 !78J0'	
 	
 	
  --fg{vSYmndovsuv%.&++2EBKKM6;;W^K_	--	
 a[!	
 	
 /86;;;NTZT_T_`gTnkmTn5Cb5H.1bkkm$.2;;=FJJtJ4LrM   r\   c                 n   U(       al  U R                   b_  [        U R                  R                  SS 5      b=  [        R
                  R                  U R                  R                  R                  5        U R                  U R                  4 H&  n[        US5      (       d  M  UR                  5         M(     g Nweightreset_parametersr   getattrr   bnrG   initzeros_r   r   r   hasattrr   r   r\   r   s      rN   init_weightsBasicBlock.init_weights  y    dmm7GDNNDUDUW_ae<f<rGGNN4>>,,334YY/Dt/00%%' 0rM   c                    UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R
                  b  XR                  U5      -   nU R                  U5      $ rh   )r   r   r   r   r   r   r   r   r   r   s      rN   r   BasicBlock.forward  u    NN1IIaLNN1NN1NN1==$MM(++Axx{rM   )r   r   r   r   r   r   r   )   r%   r%   r%   Nr9   r   TFNNr   NNF)rA   rB   rC   rD   rE   rJ   r   r
   rK   rF   rp   r   r   r   r   r   rL   r   r   s   @rN   r   r   U  s      !(.(,"%#"$"#'$&!1M1M 1M 	1M
 1M CHo1M !1M  1M 1M 1M 1M 1M !1M "1M 1Mf(4 (	 	rM   r   c                      ^  \ rS rSrSr               SS\S\S\S\S\\\4   S\S	\\   S
\	S\
S\
S\
S\
S\S\S\4U 4S jjjrSS\
4S jjrS rSrU =r$ )BottleneckBlocki  z3ResNet-like Bottleneck Block - 1x1 - kxk - 1x1
    r   r   r   r   r   r   r   rS   r   r   
extra_conv	bottle_inr   r   r   c                   > UUS.n[         TU ]  5         U=(       d
    [        5       n[        U(       a  UOUU-  5      n[	        UU5      n[        UUU4UUSUS.UD6U l        UR                  " UUS40 UD6U l        UR                  " UUU4UUS   UUS.UD6U l	        U(       a!  UR                  " UUU4US   US.UD6U l
        O[        R                  " 5       U l
        U	(       d  UR                  c  [        R                  " 5       OUR                  " U40 UD6U l        UR                  " UUS4SS0UD6U l        U	(       a  UR                  c  [        R                  " 5       OUR                  " U40 UD6U l        US	:  a  [!        U5      O[        R                  " 5       U l        U
(       a  [        R                  " 5       U l        g UR%                  S
S9U l        g )Nr   Fr   r%   r   r   r   rr   r   )r   rr   r   r   Tr   )r   r   r   r#   r   r   r   r   	conv1_1x1r   
conv2b_kxkrG   r   r   	conv3_1x1r   r   r   r   )r   r   r   r   r   r   r   r   rS   r   r   r  r  r   r   r   r   r   r   r   rr   r   s                        rN   r   BottleneckBlock.__init__  s   ( /$79 I&7l!RSJ0'	
 	
 	
  --fgqGBG--	
 a[!	
 	
 $22 "! DO !kkmDO%.&++2EBKKM6;;W^KebdKe	--gwYUYVXY.76;;;NTZT_T_`gTnkmTn5Cb5H.1bkkm$.2;;=FJJtJ4LrM   r\   c                 n   U(       al  U R                   b_  [        U R                  R                  SS 5      b=  [        R
                  R                  U R                  R                  R                  5        U R                  U R                  4 H&  n[        US5      (       d  M  UR                  5         M(     g r   )r   r   r  r   rG   r   r   r   r   r   r   r   r   s      rN   r   BottleneckBlock.init_weights  r   rM   c                 V   UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  U5      nU R                  b  XR                  U5      -   nU R                  U5      $ rh   )	r	  r   r
  r   r  r   r   r   r   r   s      rN   r   BottleneckBlock.forward  s    NN1NN1OOAIIaLNN1NN1NN1==$MM(++Axx{rM   )	r   r   r   r	  r   r
  r  r   r   )r   r%   r  r9   Nr   FFFFNNr   NNr  rA   rB   rC   rD   rE   rJ   r   rK   r
   rF   rp   r   r   r   r   r   rL   r   r   s   @rN   r  r    s	     !(."$(,##$$#"#'$&%?M?M ?M 	?M
 ?M CHo?M  ?M !?M ?M ?M ?M ?M ?M ?M !?M  "!?M ?MB(4 ( rM   r  c                      ^  \ rS rSrSr             SS\S\S\S\S\\\4   S\S	\\   S
\	S\
S\
S\S\S\4U 4S jjjrSS\
4S jjrS rSrU =r$ )	DarkBlocki  a  DarkNet-like (1x1 + 3x3 w/ stride) block

The GE-Net impl included a 1x1 + 3x3 block in their search space. It was not used in the feature models.
This block is pretty much a DarkNet block (also DenseNet) hence the name. Neither DarkNet or DenseNet
uses strides within the block (external 3x3 or maxpool downsampling is done in front of the block repeats).

If one does want to use a lot of these blocks w/ stride, I'd recommend using the EdgeBlock (3x3 /w stride + 1x1)
for more optimal compute.
r   r   r   r   r   r   r   rS   r   r   r   r   r   c           	        > XS.n[         TU ]  5         U=(       d
    [        5       n[        X&-  5      n[	        UU5      n[        UUU4UUSUS.UD6U l        UR                  " UUS40 UD6U l        U	(       d  UR                  c  [        R                  " 5       OUR                  " U40 UD6U l	        UR                  " UUU4UUS   UUSS.UD6U l        U	(       a  UR                  c  [        R                  " 5       OUR                  " U40 UD6U l        US:  a  [        U5      O[        R                  " 5       U l        U
(       a  [        R                  " 5       U l        g UR!                  SS	9U l        g )
Nr   Fr   r%   r   r   r   rr   r   r   r   Tr   )r   r   r   r#   r   r   r   r   r	  r   rG   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rS   r   r   r   r   r   r   r   r   r   rr   r   s                      rN   r   DarkBlock.__init__  s_   $ /$79 !78J0'	
 	
 	
  --fgqGBG%.&++2EBKKM6;;W^KebdKe	--

 a[!

 

 /86;;;NTZT_T_`gTnkmTn5Cb5H.1bkkm$.2;;=FJJtJ4LrM   r\   c                 n   U(       al  U R                   b_  [        U R                  R                  SS 5      b=  [        R
                  R                  U R                  R                  R                  5        U R                  U R                  4 H&  n[        US5      (       d  M  UR                  5         M(     g r   r   r   s      rN   r   DarkBlock.init_weights8  r   rM   c                    UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R
                  b  XR                  U5      -   nU R                  U5      $ rh   )r	  r   r   r   r   r   r   r   s      rN   r   DarkBlock.forward?  r   rM   )r   r   r   r	  r   r   r   )r   r%   r  r9   Nr   TFNNr   NNr  r  r   s   @rN   r  r    s      !(."%(,#"$"#'$&!2M2M 2M 	2M
 2M CHo2M  2M !2M 2M 2M 2M 2M !2M "2M 2Mh(4 (	 	rM   r  c                      ^  \ rS rSrSr             SS\S\S\S\S\\\4   S\S	\\   S
\	S\
S\
S\S\S\4U 4S jjjrSS\
4S jjrS rSrU =r$ )	EdgeBlockiK  a  EdgeResidual-like (3x3 + 1x1) block

A two layer block like DarkBlock, but with the order of the 3x3 and 1x1 convs reversed.
Very similar to the EfficientNet Edge-Residual block but this block it ends with activations, is
intended to be used with either expansion or bottleneck contraction, and can use DW/group/non-grouped convs.

FIXME is there a more common 3x3 + 1x1 conv block to name this after?
r   r   r   r   r   r   r   rS   r   r   r   r   r   c                   > XS.n[         TU ]  5         U=(       d
    [        5       n[        X&-  5      n[	        UU5      n[        UUU4UUSUS.UD6U l        UR                  " UUU4UUS   UUS.UD6U l        U	(       d  UR                  c  [        R                  " 5       OUR                  " U40 UD6U l	        UR                  " UUS4SS0UD6U l        U	(       a  UR                  c  [        R                  " 5       OUR                  " U40 UD6U l        US:  a  [        U5      O[        R                  " 5       U l        U
(       a  [        R                  " 5       U l        g UR!                  S	S
9U l        g )Nr   Fr   r   r  r%   r   r   Tr   )r   r   r   r#   r   r   r   r   r   r   rG   r   	conv2_1x1r   r   r   r   r  s                      rN   r   EdgeBlock.__init__U  sb   $ /$79 !78J0'	
 	
 	
  --	
 a[!	
 	
 &/&++2EBKKM6;;W^KebdKe	--gwYUYVXY.76;;;NTZT_T_`gTnkmTn5Cb5H.1bkkm$.2;;=FJJtJ4LrM   r\   c                 n   U(       al  U R                   b_  [        U R                  R                  SS 5      b=  [        R
                  R                  U R                  R                  R                  5        U R                  U R                  4 H&  n[        US5      (       d  M  UR                  5         M(     g r   )r   r   r  r   rG   r   r   r   r   r   r   r   r   s      rN   r   EdgeBlock.init_weights  r   rM   c                    UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R
                  b  XR                  U5      -   nU R                  U5      $ rh   )r   r   r  r   r   r   r   r   s      rN   r   EdgeBlock.forward  r   rM   )r   r   r   r   r  r   r   )r   r%   r  r9   Nr   FFNNr   NNr  r  r   s   @rN   r  r  K  s      !(."%(,##$"#'$&!0M0M 0M 	0M
 0M CHo0M  0M !0M 0M 0M 0M 0M !0M "0M 0Md(4 (	 	rM   r  c                   *  ^  \ rS rSrSr            SS\S\S\S\S\\\4   S\S	\\   S
\	S\
S\S\S\4U 4S jjjrSS\4S jjrS rS rS\\R$                  \R$                  4   4S jrS\\R$                  \R$                  4   4S jrSrU =r$ )RepVggBlocki  zHRepVGG Block.

Adapted from impl at https://github.com/DingXiaoH/RepVGG
r   r   r   r   r   r   r   rS   r   r   r   inference_modec                   > XS.n[         TU ]  5         [        Xq5      =U l        nU	=(       d
    [	        5       n	U(       a$  [
        R                  " SUUUUUUSS.UD6U l        OS U l        X:H  =(       a    US:H  =(       a    US   US   :H  nU(       a  U	R                  " U4SS0UD6OS U l	        U	R                  " UUU4UUS   UU
SS.UD6U l        U	R                  " UUS4UUSS	.UD6U l        US
:  a  U(       a  [        U5      O[
        R                  " 5       U l        U	R                   c  [
        R                  " 5       OU	R                   " U40 UD6U l        U	R#                  SS9U l        g )Nr   Tin_channelsout_channelsr   r   r   rr   biasr%   r   r   Fr  )r   rr   r   r   r   r@   )r   r   r   rr   r   rG   Conv2dreparam_convr   identityr   conv_kxkconv_1x1r   r   r   r   r   )r   r   r   r   r   r   r   r   rS   r   r   r   r'  r   r   r   rr   	use_identr   s                     rN   r   RepVggBlock.__init__  s   " /)*==f$79 "		 	!"$'!	! 	!D !%D)XfkXhqkXVW[>XIOXFOOGKuKK^bDM"00
 !!%
 
DM #00  DM :H"9LQZXn5`b`k`k`mDN%+[[%8BKKMfkk'>XUW>X	::d:+rM   r\   c                    U R                  5        Hz  n[        U[        R                  5      (       d  M$  [        R                  R                  UR                  SS5        [        R                  R                  UR                  SS5        M|     [        U R                  S5      (       a  U R                  R                  5         g g )Ng?r   r   )modulesr   rG   BatchNorm2dr   normal_r   r,  r   r   r   )r   r\   ms      rN   r   RepVggBlock.init_weights  s|    A!R^^,,"b12.   499011II&&( 2rM   c                    U R                   b/  U R                  U R                  U R                  U5      5      5      $ U R                  c$  U R	                  U5      U R                  U5      -   nOIU R                  U5      nU R	                  U5      U R                  U5      -   nU R                  U5      nX-  nU R                  U5      nU R                  U5      $ rh   )r.  r   r   r/  r1  r0  r   )r   r   r/  s      rN   r   RepVggBlock.forward  s    (88DIId&7&7&:;<<== a 4==#33A}}Q'Ha 4==#33Aq!AMAIIaLxx{rM   c                 z   U R                   b  gU R                  5       u  p[        R                  " U R                  R
                  R                  U R                  R
                  R                  U R                  R
                  R                  U R                  R
                  R                  U R                  R
                  R                  U R                  R
                  R                  U R                  R
                  R                  SS9U l         XR                   R                  l        X R                   R                  l        U R!                  5        H  u  p4SU;   a  M  UR#                  5         M     U R%                  S5        U R%                  S5        U R%                  S5        U R%                  S5        g)	Following works like `RepVGG: Making VGG-style ConvNets Great Again` -
https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched
architecture used at training time to obtain a plain CNN-like structure
for inference.
NTr*  r+  r   r   paddingr   rr   r,  r.  r0  r1  r/  r   r.  _get_kernel_biasrG   r-  r0  r   r*  r+  r   r   r?  r   rr   r   datar,  named_parametersdetach___delattr__r   kernelr,  nameparas        rN   reparameterizeRepVggBlock.reparameterize  sG    (,,.II**66++88**66==%%,,MM&&..]]''00==%%,,	
 )/  %&*# //1JD%LLN 2 	$$$%rM   rs   c                    SnSnU R                   bo  U R                  U R                   5      u  pU R                  R                  R                  S   S-  n[
        R                  R                  R                  XX3U/5      nSnSnU R                  b  U R                  U R                  5      u  pEU R                  U R                  5      u  pgXa-   U-   nXr-   U-   n	X4$ zzMethod to obtain re-parameterized kernel and bias.
Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L83
r   r6   )
r1  _fuse_bn_tensorr0  r   r   r   rG   
functionalpadr/  )
r   
kernel_1x1bias_1x1rP  kernel_identitybias_identitykernel_conv	bias_convkernel_final
bias_finals
             rN   rA  RepVggBlock._get_kernel_bias  s    
 
==$#'#7#7#F J--$$003q8C,,003S=QRJ ==$-1-A-A$---P*O "&!5!5dmm!D"//A)M9
''rM   c                    [        U[        5      (       a  UR                  R                  nUR                  R
                  nUR                  R                  nUR                  R                  nUR                  R                  nUR                  R                  nGO*[        U[        R                  5      (       d   e[        U S5      (       d  U R                  R                  R                  nXR                  -  n	U R                  R                  R                  n
[         R"                  " U R                  R                  R                  5      n[%        U5       H  nSXX-  U
S   S-  U
S   S-  4'   M     Xl        U R&                  nUR
                  nUR                  nUR                  nUR                  nUR                  nXG-   R)                  5       nX]-  R+                  SSSS5      nX.-  XcU-  U-  -
  4$ )Method to fuse batchnorm layer with preceding conv layer.
Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L95
	id_tensorr%   r   r6   r   r   r   r   r   running_meanrunning_varr,  epsrG   r6  r   r0  r*  rr   r   r   
zeros_liker   r\  sqrtreshaper   branchrG  r_  r`  gammabetara  r   	input_dimr   kernel_valuer   stdts                  rN   rN  RepVggBlock._fuse_bn_tensor0  s    fk**[[''F!9911L ))//KII$$E99>>D))--Cfbnn55554--++77"kk1	"mm00<<$//0B0B0I0IJvA_`LAM;q>Q3FTUZ[H[![\ '!-^^F!..L ,,KMME;;D**C &&([!!"aA.z4"6"<<<<rM   )	r   r   r1  r0  r   rr   r\  r/  r.  )r   r%   r  r9   Nrb   NNr   FNNr  )rA   rB   rC   rD   rE   rJ   r   rK   r
   rF   r   r   rp   r   r   r   rJ  r   r   rA  rN  rL   r   r   s   @rN   r&  r&    s     !(."%(, "#'$&#(<,<, <, 	<,
 <, CHo<,  <, !<, <, <, !<, "<, !<, <,|)4 )&B(%ell(B"C (4=u||U\\/I)J = =rM   r&  c                   V  ^  \ rS rSrSr             SS\S\S\S\S\\\4   S	\S
\\   S\	S\
S\S\S\S\SS4U 4S jjjrS\R                  S\R                  4S jrS rS\\R                  \R                  4   4S jrS\\R                  \R                  4   4S jrSrU =r$ )MobileOneBlockiP  a  MobileOne building block.

This block has a multi-branched architecture at train-time
and plain-CNN style architecture at inference time
For more details, please refer to our paper:
`An Improved One millisecond Mobile Backbone` -
https://arxiv.org/pdf/2206.04040.pdf
Nr   r   r   r   r   r   r   rS   r'  r   r   r   r   rs   c                   > XS.n[         TU ]  5         Xl        [        Xq5      =U l        nU=(       d
    [        5       nU	(       a%  [        R                  " SUUUUUUSS.UD6U l        GO
SU l        X:H  =(       a    US:H  =(       a    US   US   :H  nU(       a  UR                  " U4SS0UD6OSU l
        / n[        U R                  5       H+  nUR                  UR                  " UU4UUUSS	.UD65        M-     [        R                  " U5      U l        SU l        US:  a  UR                  " UU4SUUSS	.UD6U l        US
:  a  U(       a  [#        U5      O[        R$                  " 5       U l        UR(                  c  [        R$                  " 5       OUR(                  " U40 UD6U l        UR+                  SS9U l        g)z+Construct a MobileOneBlock module.
        r   Tr)  Nr%   r   r   F)r   r   rr   r   r   r   r@   )r   r   r   r   rr   r   rG   r-  r.  r   r/  r   appendr   
ModuleListr0  
conv_scaler   r   r   r   r   )r   r   r   r   r   r   r   r   rS   r'  r   r   r   r   r   r   r   rr   r2  convsr   r   s                        rN   r   MobileOneBlock.__init__Z  s   ( /!2)*==f$79 "		 	!"$'!	! 	!D !%D )XfkXhqkXVW[>XIOXFOOGKuKK^bDM E4112V11 !,!!#   3 MM%0DM #DOQ"("6"6# !"!!## # :H"9LQZXn5`b`k`k`mDN%+[[%8BKKMfkk'>XUW>X	::d:+rM   r   c                    U R                   b/  U R                  U R                  U R                  U5      5      5      $ SnU R                  b  U R                  U5      nSnU R                  b  U R	                  U5      nUnU R
                   H  nXE" U5      -  nM     U R                  U5      nXB-  nU R                  U R                  U5      5      $ )zApply forward pass. r   )r.  r   r   r/  rs  r0  r   )r   r   identity_out	scale_outoutcks         rN   r   MobileOneBlock.forward  s     (88DIId&7&7&:;<< ==$==+L 	??&*I --B2a5LC  nnS!xx		#''rM   c                    U R                   b  gU R                  5       u  p[        R                  " U R                  S   R
                  R                  U R                  S   R
                  R                  U R                  S   R
                  R                  U R                  S   R
                  R                  U R                  S   R
                  R                  U R                  S   R
                  R                  U R                  S   R
                  R                  SS9U l         XR                   R                  l        X R                   R                  l        U R!                  5        H  u  p4SU;   a  M  UR#                  5         M     U R%                  S5        U R%                  S5        U R%                  S5        U R%                  S	5        g)
r=  Nr   Tr>  r.  r0  rs  r/  r   r@  rF  s        rN   rJ  MobileOneBlock.reparameterize  sj    (,,.IIa(--99q)..;;a(--99==#((//MM!$))11]]1%**33==#((// )/  %&*# //1JD%LLN 2 	$&$%rM   c                    SnSnU R                   br  U R                  U R                   5      u  pU R                  S   R                  R                  S   S-  n[
        R                  R                  R                  XX3U/5      nSnSnU R                  b  U R                  U R                  5      u  pESnSn[        U R                  5       H+  nU R                  U R                  U   5      u  pXi-  nXz-  nM-     Xa-   U-   nXr-   U-   nX4$ rM  )rs  rN  r0  r   r   r   rG   rO  rP  r/  r   r   )r   kernel_scale
bias_scalerP  rS  rT  rU  rV  ix_kernel_biasrW  rX  s                rN   rA  MobileOneBlock._get_kernel_bias  s   
 
??&'+';';DOO'L$L--"''33A6!;C 88..22<sQTAUVL ==$-1-A-A$---P*O 	../B!11$--2CDNG"KI 0
 #1OC+m;
''rM   c                 
   [        U[        5      (       a  UR                  R                  nUR                  R
                  nUR                  R                  nUR                  R                  nUR                  R                  nUR                  R                  nGO3[        U[        R                  5      (       d   e[        U S5      (       d  U R                  S   R                  R                  nXR                  -  n	U R                  S   R                  R                  n
[         R"                  " U R                  S   R                  R                  5      n[%        U5       H  nSXX-  U
S   S-  U
S   S-  4'   M     Xl        U R&                  nUR
                  nUR                  nUR                  nUR                  nUR                  nXG-   R)                  5       nX]-  R+                  SSSS5      nX.-  XcU-  U-  -
  4$ )r[  r\  r   r%   r6   r]  r^  re  s                  rN   rN  MobileOneBlock._fuse_bn_tensor  s    fk**[[''F!9911L ))//KII$$E99>>D))--Cfbnn55554--q)..::"kk1	"mmA.33??$//a0@0E0E0L0LMvA_`LAM;q>Q3FTUZ[H[![\ '!-^^F!..L ,,KMME;;D**C &&([!!"aA.z4"6"<<<<rM   )
r   r   r0  rs  r   rr   r\  r/  r   r.  )r   r%   r  r9   Nrb   Fr%   NNr   NN)rA   rB   rC   rD   rE   rJ   r   rK   r
   rF   rp   r   r   r   r   r   r   rJ  rA  rN  rL   r   r   s   @rN   ro  ro  P  sD     !(."%(, #(%&"#'$&!I,I, I, 	I,
 I, CHoI,  I, !I, I, !I,  #I, I, !I, "I," 
#I, I,V( (%,, (4&@(%ell(B"C (>=u||U\\/I)J = =rM   ro  c            !          ^  \ rS rSrSr                SS\S\S\S\S\\\4   S\S	\\   S
\	S\
S\
S\
S\
S\\\\4      S\S\S\4 U 4S jjjrSS\
4S jjrS rSrU =r$ )SelfAttnBlocki  zHResNet-like Bottleneck Block - 1x1 - optional kxk - self attn - 1x1
    r   r   r   r   r   r   r   rS   r  r   r  post_attn_na	feat_sizer   r   r   c                 "  > UUS.n[         TU ]  5         Uc   e[        U(       a  UOUU-  5      n[        UU5      n[	        UUU4UUSUS.UD6U l        UR                  " UUS40 UD6U l        U	(       a%  UR                  " UUU4UUS   UUS.UD6U l        SnO[        R                  " 5       U l        Uc  0 O[        US9nUR                  " U4SU0UDUD6U l        U(       a  UR                  " U40 UD6O[        R                  " 5       U l        UR                  " UUS4S	S0UD6U l        US
:  a  [!        U5      O[        R                  " 5       U l        U
(       a  [        R                  " 5       U l        g UR%                  SS9U l        g )Nr   Fr   r%   r   r  )r  r   r   r   Tr   )r   r   r#   r   r   r   r   r	  r   rG   r   rj   r   r   	post_attnr  r   r   r   )r   r   r   r   r   r   r   r   rS   r  r   r  r  r  r   r   r   r   r   r   r   rr   
opt_kwargsr   s                          rN   r   SelfAttnBlock.__init__"  s   * /!!! I&7l!RSJ0'	
 	
 	
  --fgqGBG#11	 !!%	 	DN F[[]DN$,R$2K
))'U&UJURTU;G7B7R[[]--gwYUYVXY5Cb5H.1bkkm$.2;;=FJJtJ4LrM   r\   c                 V   U(       al  U R                   b_  [        U R                  R                  SS 5      b=  [        R
                  R                  U R                  R                  R                  5        [        U R                  S5      (       a  U R                  R                  5         g g r   )r   r   r  r   rG   r   r   r   r   r   r   )r   r\   s     rN   r   SelfAttnBlock.init_weights_  so    dmm7GDNNDUDUW_ae<f<rGGNN4>>,,3344>>#566NN++- 7rM   c                 4   UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  b  XR                  U5      -   nU R                  U5      $ rh   )r	  r   r   r  r  r   r   r   r   s      rN   r   SelfAttnBlock.forwarde  s    NN1NN1NN1NN1NN1NN1==$MM(++Axx{rM   )r   r	  r   r  r   r  r   r   )r   r%   r  r9   Nr   FFFTNNNr   NNr  r  r   s   @rN   r  r    s$     !(."$(,#$$#!%37"#'$&';M;M ;M 	;M
 ;M CHo;M  ;M !;M ;M ;M ;M ;M ;M  c3h0;M ;M  !!;M" "#;M ;Mz.4 .
 
rM   r  )basicbottledarkedger~   r   r   r   block_fnc                     U[         U '   g rh   )_block_registry)r   r  s     rN   register_blockr  }  s    "*OJrM   blockc                     [        U [        R                  [        45      (       a  U " S0 UD6$ U [        ;   d
   SU  35       e[        U    " S0 UD6$ )NzUnknown block type (r@   )r   rG   rH   r   r  )r  r   s     rN   r0   r0     sR    %"))W-..vO#C';E7%CC#5!+F++rM   c                      ^  \ rS rSr         SS\S\\\\   \\S4   4   S\S\S\S\S	\	\   S
\
S\4U 4S jjjrS\\R                  \	\R                     4   4S jrSrU =r$ )Stemi  r   r   .r   r   r   num_repnum_act	chs_decayr   c           
      l  > XS.n[         TU ]  5         US;   d   eU	=(       d
    [        5       n	[        U[        [
        45      (       a  [        U5      nUnO0[        U5       Vs/ s H  n[        X(U-  -  5      PM     snS S S2   nX@l	        / U l
        SnS/S/US-
  -  -   nUS:X  a  U(       d  SUS'   Uc  UOUnS/Xg-
  -  S	/U-  -   nUnSnSn[        [        UUU5      5       H  u  nu  nnnU(       a  U	R                  O[        nS
US-    3nUS:  a0  US:  a*  US-
  nU R                  R                  [!        UUUSS95        U R#                  UU" UU4UUS.UD65        UnUU-  nUnM     U(       a  UR%                  5       nUS;   d   eWnU R                  R                  [!        UUUSS95        US:X  a'  U R#                  S[&        R(                  " S5      5        OUS:X  a'  U R#                  S[&        R*                  " S5      5        OZSU;   a'  U R#                  S[&        R(                  " SSSS95        O-SU;   a'  U R#                  S[&        R*                  " SSSSS95        US-  nSnUS:  a  UOS U l        U R                  R                  [!        UUUSS95        UU:X  d   eg s  snf )Nr   )r6      r]  rb   r6   r%   r  FTr   r   num_chs	reductionmodulestage)r   r   )maxrV   r   avgpoolmax2avg2r  r   r  r  r   )r   r   r?  r   )r   r   r?  r   )r   r   r   r   r   r   r   r   roundr   feature_infor   r   r   r   rq  rj   
add_modulelowerrG   	MaxPool2dr   last_feat_idx)r   r   r   r   r   r   r  r  r  r   r   r   r   rY   r   	prev_featstem_stridesstem_norm_actsprev_chscurr_strider  chr7   nalayer_fn	conv_namer   s                             rN   r   Stem.__init__  s    /$79ge}--'lGHAFwPAgQ67PQUSUQUVH	saSGaK00Q;t L$_''G$56$'9II'HlN(STNA{Ar/1v++}Hq1ugI1uQ !A!!((h+^gop)qrOOIx"'f+^_'fce'fgH1K!I U ::<DOOOOM$$T(kZckl%mnv~Q8Q8$1VW(XY$1VWkp(qr1KI.;q.@]d  h+V_gh!ijf$$$[ Qs   %J1rs   c                     S n[        U 5       H/  u  p4U" U5      nU R                  c  M  X0R                  :X  d  M-  UnM1     X4$ rh   )r   r  )r   r   intermediater   r8  s        rN   forward_intermediatesStem.forward_intermediates  sF    /3dODA!A!!-!7I7I2I  $ rM   )r  r  r   )	r   r  rV   r   N      ?NNN)rA   rB   rC   rD   rJ   r   r   r   rF   r
   rK   r   r   r   r   r  rL   r   r   s   @rN   r  r    s      !!%)""D%D% 3S	5c?:;D% 	D%
 D% D% D% c]D% D% D% D%L%hu||>T0T*U  rM   r  rb   rU   	pool_typefeat_prefixc                 2   XgS.nU=(       d
    [        5       nUS;   d   eSU;   a  SU;   a  SOS n	[        X4SXUS.UD6n
OSU;   a  [        U S	U-  S
-  US-  U44X5S.UD6n
OSU;   a  [        X4S	SX5S.UD6n
OSU;   a  [        X4SUS.UD6n
OSU;   a  [        X4S	SUS.UD6n
OSU;   a/  U(       a  [        XS4SX5S.UD6n
OpUR                  " XS4SS0UD6n
OY[        U[        [        45      (       a  [        XS	4X5S.UD6n
O.U(       a  [        XS	4SX5S.UD6n
OUR                  " XS	4SS0UD6n
[        U
[        5      (       a:  U
R                   Vs/ s H   n[        USR                  XKS   /5      S9PM"     nnX4$ [        USUSS9/nX4$ s  snf )Nr   )	rb   quadquad2tiereddeepr~   r   7x7rT   r  r  r6   r  )r  r  r   r   r  r      )r   r   r  r9   )r  r  r   r   r~   )r   r   r   )r   r   r   r     r%   )r  r   r   r   .r  )r  r   r  )r   r  r&  ro  r   r   r   r   r  rj   join)r   r   rU   r  r  r   r   r   r   r  stemfr  s                rN   r/   r/     s    	+B wyF[[[[)+!FeQX^ebde	Y	FQ[A-w!|WEkIkhjk	9	FcQ#Ic`bc	)	6J1VJrJ	)	f\1Qv\Y[\	)	[AI[XZ[D''K1KKDgt}--PPRPD FQ_	_\^_++FQOqOBO$VZVgVghVgQRQsxxk0J'KLVgh  W+UVWX is   'Fr6   c                 R    U c  S $ [        U  Vs/ s H  o"U-  PM	     sn5      $ s  snf rh   )r   )r  r   r7   s      rN   reduce_feat_sizer    s+    $4Q%i0Pifi0P*QQ0Ps   $c                 &    U b  U OUnU=(       d    0 $ )a!  Override model level attn/self-attn/block kwargs w/ block level

NOTE: kwargs are NOT merged across levels, block_kwargs will fully replace model_kwargs
for the block if set to anything that isn't None.

i.e. an empty block_kwargs dict will remove kwargs set at model level for that block
r@   )r?   model_kwargs
out_kwargss      rN   override_kwargsr    s     ".!9|JrM   r?   	block_cfg	model_cfgc                    U S   nUR                   S LnU(       d  UR                  b}  U(       a  UR                   (       d  S nOY[        UR                  UR                  5      nUR                   =(       d    UR                   nUb  [        [	        U5      40 UD6OS n[        X5S9nUR                  S LnU(       d  UR                  b}  U(       a  UR                  (       d  S nOY[        UR                  UR                  5      n	UR                  =(       d    UR                  nUb  [        [	        U5      40 U	D6OS n[        X8S9nX0S'   U R                  [        UR                  UR                  5      5        g )Nr   )r   r   )
r;   r<   r  r   r"   r   r=   r>   updater?   )
r?   r  r  	layer_fnsattn_setr;   r<   self_attn_setr=   r>   s
             rN   update_block_kwargsr    sG   X&I ##4/H9((4I00J))*?*?AVAVWK"--E1E1EJISI_*!5EEeiJI7	 --T9M	22>!:!:"O.y/I/I9KeKef'77T9;T;TO". &h&?TCST48 IA	& 	(>(>	@V@VWXrM   r   r   r  	drop_prob
block_size
num_stagesc                     US:  d   eS/U-  nU (       a*  [        [        XS-  S-
  SS9US'   [        [        XSS9US'   U$ )	aK  Create DropBlock layer partials for each stage.

DropBlock is applied to the last two stages only, following common practice.
The block_size specifies the size for the final stage; the second-to-last
stage uses a larger block size scaled to account for 2x larger feature maps.

Args:
    drop_prob: Drop probability for DropBlock.
    block_size: Block size for the final stage. Second-to-last stage
        uses `block_size * 2 - 1` to scale with feature map size.
    num_stages: Number of stages in the model.

Returns:
    List of DropBlock partial instances or None for each stage.
r6   Nr%         ?)r  r  gamma_scaler9   r]  )r   r   )r  r  r  dbss       rN   drop_blocksr  ?  sU    ( ??&:
C+TU~XYGYgklB+_cdBJrM   r   r   output_stride	stem_featdrop_block_ratedrop_block_sizer  block_kwargs_fnc                    U=(       d
    [        5       n/ nU R                   Vs/ s H  n[        U5      PM     nn[        U5      nU VVs/ s H)  n[	        U Vs/ s H  nUR
                  PM     sn5      PM+     nnn[        UUSS9n[        XEU5      nSnUS   nUS   nUn/ n[        U5       GHt  u  nnUS   R                  nUS:w  a  U(       a  UR                  U5        UU:  a  US:  a  UU-  nSnUU-  nUS;   a  SOSn/ n[        U5       H  u  nn[        UR                  U R                  -  5      n UR                  n![        U![         5      (       a	  U!" U U5      n![#        UU US:X  a  UOSUU4U!UR$                  U R&                  UU   UU   U   UU	U
S	9n"UR(                  S
;   a  UU"S'   U" U"UU S9  U[+        UR(                  40 U"D6/-  nUnU nUS:  d  M  US:X  d  M  [-        UU5      nM     U[.        R0                  " U6 /-  n[#        UUSU 3US-   S9nGMw     UR                  U5        [.        R0                  " U6 X4$ s  snf s  snf s  snnf )NT)	stagewiser%   r  r  r   )r%   r6   r6   )r   r   r   r   r   r   rS   r   r   r   r   r   r  r  )r  r  stages.r  )r   rQ   r   r   sumr4   r   r  r   r7   rq  r#   r5   rZ   r8   r   r   rj   r:   rS   r3   r0   r  rG   
Sequential)#r   r   r  r  r  r  r  r   r  r   r   r  r7   r   r  	stage_bcsbcdepthsdprr  r   
net_strider  r  stages	stage_idxstage_block_cfgsr   first_dilationrQ   	block_idxr  r   r   r?   s#                                      rN   create_byob_stagesr  \  s     wyFL03

;
1#A&
J;ZJ?IJz)c),)B244),-zFJ
#NFd
KC
o

CCH;'J#HIF'0'<#	#!!$&&Q;9	*&6A:HFf
&&0a$-.>$? Iy$Y[[33C3C%CDG"J*h//';
!*avQ((3%&\\>>y>"9~i8L ~~/,5[)LIM|INNClCDDF%NHzi1n,Y?	7 %@: 	2==&)**Z'R[Q\H]enqrers	S (=V 	"==&!<::m <,Js   III0IITallow_aac                    [        U R                  5      n[        U R                  US9nU R                  (       a+  U(       a$  [        [        U R                  X R                  S9nO[        [        U R                  US9nU R                  (       a)  [        [        U R                  5      40 U R                  D6OS nU R                  (       a)  [        [        U R                  5      40 U R                  D6OS n[        XCX%US9nU$ )N)ra   r_   )ra   r_   rc   )r   r   r   r   r   )r    r_   r!   ra   rc   r   r   r;   r"   r<   r=   r>   r   )r   r  r   r   r   r   r   r  s           rN   get_layer_fnsr    s    

&C!S^^sKH
||RU`l`lmRUVCF>>78CNN+?s?W[DRUReRe!4!45N9M9MNkoI]3enoHOrM   c                   V  ^  \ rS rSrSr            S'S\S\S\S\\   S\S	\\	\\
\\4   4      S
\S\S\S\S\4U 4S jjjr\R                  R                   S(S\S\\\4   4S jj5       r\R                  R                   S)S\SS4S jj5       r\R                  R                   S\R,                  4S j5       rS*S\S\\   SS4S jjr      S+S\R2                  S\\	\\\   4      S\S\S\S\S\S\	\\R2                     \
\R2                  \\R2                     4   4   4S jjr   S,S\	\\\   4   S\S \S\\   4S! jjrS\R2                  S\R2                  4S" jrS(S\R2                  S#\S\R2                  4S$ jjrS\R2                  S\R2                  4S% jrS&r U =r!$ )-r,   i  a  Bring-your-own-blocks Network.

A flexible network backbone that allows building model stem + blocks via
dataclass cfg definition w/ factory functions for module instantiation.

Current assumption is that both stem and blocks are in conv-bn-act order (w/ block ending in act).
Nr   num_classesin_chansglobal_poolr  img_size	drop_rater  r  r   r\   c           
      B	  > [         TU ]  5         XS.nX l        X0l        Xpl        SU l        [        U40 UD6n[        USS9n[        U5      nUR                  (       a
  Uc   S5       eUb  [        U5      OSn/ U l
        [        UR                  [        [        45      (       a<  UR                   Vs/ s H$  n[        [!        UUR"                  -  5      5      PM&     nnOK[        [!        UR                  =(       d    UR$                  S   R&                  UR"                  -  5      5      n[)        SUUUR*                  UR,                  US.UD6u  U l        nU R                  R1                  USS 5        [3        UUS   S	   S
9n[5        UU
UUS   4UU	UUS.UD6u  U l        nnU R                  R1                  USS 5        US   S	   nUS   S   nUR8                  (       aU  [        [!        UR"                  UR8                  -  5      5      U l        UR:                  " UU R8                  S40 UD6U l        O!UU l        [>        R@                  " 5       U l        U =R                  [C        U R8                  US[E        U R6                  5      S9/-  sl
        U R                   Vs/ s H  nUS   PM
     snU l#        U R8                  U l$        URJ                  S;   d   eURJ                  S:X  al  Uc  Sn[M        U R8                  U4URH                  UURN                  URP                  U R                  S.UD6U l)        U RR                  RT                  U l$        GOURJ                  S:X  a_  Uc  SnUS;   d   e[W        U R8                  4URH                  UUUU R                  SS.UD6U l)        U RR                  RX                  U l$        OURJ                  S:X  a_  Uc  SnUS;   d   e[[        U R8                  4URH                  UUUU R                  SS.UD6U l)        U RR                  RX                  U l$        O=Uc  SnURH                  b   e[]        U R8                  U4UU R                  S.UD6U l)        X@l/        [a        [c        [d        US9U 5        gs  snf s  snf )a  
Args:
    cfg: Model architecture configuration.
    num_classes: Number of classifier classes.
    in_chans: Number of input channels.
    global_pool: Global pooling type.
    output_stride: Output stride of network, one of (8, 16, 32).
    img_size: Image size for fixed image size models (i.e. self-attn).
    drop_rate: Classifier dropout rate.
    drop_block_rate: DropBlock drop rate.
    drop_block_size: DropBlock block size for final stage (scales up for earlier stages).
    drop_path_rate: Stochastic depth drop-path rate.
    zero_init_last: Zero-init last weight of residual path.
    **kwargs: Extra kwargs overlayed onto cfg.
r   F)r  Nz8img_size argument is required for fixed input size modelr   )r   r   rU   r  r   r]  r  )r   )r  r  r   r  r  r%   
final_convr  r  )rb   re   mlpattn_absattn_rotr  r   )hidden_sizer  ra   r_   r  r	  token)rb   r  T)	embed_dimout_featuresr  r  r  qkv_separater
  )r  r  ref_feat_sizer  r  r  )r  r  r\   r@   )3r   r   r  r  r  grad_checkpointingr   r  r]   r$   r  r   rY   r   r   rJ   r  rZ   rQ   r5   r/   rU   rW   r  extendr  r  r  r[   r   r  rG   r   rj   r   
stage_endsrd   rf   r   ra   r_   headr  r   r  r   r   r  r(   r   _init_weights)r   r   r  r  r  r  r  r  r  r  r   r\   r   r   r   r   stem_layersstage_layersr  r5   rY   r  
stage_featr  r  r  r   s                             rN   r   ByobNet.__init__  s   @ 	/& ""'c$V$#C%8$S)'c)cc'+3+?Ih'T	cllT5M22BE,,O,QE!c&6&6"678,HOH5#,,"A#**Q-//SEUEU!UVWH/  
mmmm 
  
	9 	  3B0$Yy}[7QR	-?bM	
.

 ,+
.
 
.
*Z 	  CR1rN;/	b>), #E#*:*:S=M=M*M$N OD*884CTCTVW^[]^DO (D kkmDO**i\_`d`k`k\lmo 	o/3/@/@A/@!1W:/@A $ 1 1}} QQQQ==E!"#-!!	  00%>>--..	 	DI %)II$9$9D!]]j("%-///'!!	..(#%..!	 	DI %)II$7$7D!]]j("%-///*!!	..('%..!	 	DI %)II$7$7D!"#''///&!! &..	
 DI ' 	GM.I4PG PJ Bs   3+R'Rcoarsers   c                 4    [        SU(       a  SOSS4S/S9nU$ )zGroup matcher for parameter groups.

Args:
    coarse: Whether to use coarse grouping.

Returns:
    Dictionary mapping group names to patterns.
z^stemz^stages\.(\d+)z^stages\.(\d+)\.(\d+)N)z^final_conv)i )r  rQ   ri   )r   r  matchers      rN   group_matcherByobNet.group_matcherO  s.     &,"2JDQ*
 rM   enablec                     Xl         g)zgEnable or disable gradient checkpointing.

Args:
    enable: Whether to enable gradient checkpointing.
N)r  )r   r   s     rN   set_grad_checkpointingByobNet.set_grad_checkpointingb  s
     #)rM   c                 .    U R                   R                  $ )z8Get classifier module.

Returns:
    Classifier module.
)r  fc)r   s    rN   get_classifierByobNet.get_classifierk  s     yy||rM   c                 F    Xl         U R                  R                  X5        g)zvReset classifier.

Args:
    num_classes: Number of classes for new classifier.
    global_pool: Global pooling type.
N)r  r  reset)r   r  r  s      rN   reset_classifierByobNet.reset_classifiert  s     '		1rM   r   indicesnorm
stop_early
output_fmtintermediates_onlyexclude_final_convc                    US;   d   S5       e/ n[        [        U R                  5      U5      u  pU	 Vs/ s H  oR                  U   PM     n	nU R                  U
   n
Sn[        U R                  S5      (       a  U R                  R                  U5      u  pOU R	                  U5      SpX;   a  UR                  Uc  UOU5        U R                  S   n[        R                  R                  5       (       d  U(       d  U R                  nOU R                  SU
 nU H  nUS-  nU R                  (       a0  [        R                  R                  5       (       d  [        UU5      nOU" U5      nU(       d  X:X  a  U R                  U5      nX;   d  Mu  UR                  U5        M     U(       a  U$ U(       a  X:X  a  U R                  U5      nX4$ s  snf )a  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    norm: Apply norm layer to compatible intermediates
    stop_early: Stop iterating over blocks when last desired intermediate hit
    output_fmt: Shape of intermediate feature outputs
    intermediates_only: Only return intermediate features
    exclude_final_conv: Exclude final_conv from last intermediate
Returns:

)NCHWzOutput shape must be NCHW.r   r  Nr]  r%   )r'   r   r  r   r  r  rq  r   jitis_scriptingr  r  r)   r  )r   r   r,  r-  r.  r/  r0  r1  intermediatestake_indices	max_indexr   feat_idxx_interlast_idxr  r  s                    rN   r  ByobNet.forward_intermediates~  s   . Y&D(DD&"6s4??7KW"U4@ALq*LAOOI.	49956688;JAw1tw#  go7C??2&99!!##:[[F[[),FEMH&&uyy/E/E/G/G"5!,!H%(*>OOA&'$$Q'    ("6"AE Bs   G
prune_norm
prune_headc                    [        [        U R                  5      U5      u  pEU R                  U   nU R                  SU U l        XPR                  S   :  a  [        R
                  " 5       U l        U(       a  U R                  SS5        U$ )a  Prune layers not required for specified intermediates.

Args:
    indices: Indices of intermediate layers to keep.
    prune_norm: Whether to prune normalization layer.
    prune_head: Whether to prune the classifier head.

Returns:
    List of indices that were kept.
Nr]  r   rb   )r'   r   r  r  rG   r   r  r*  )r   r,  r=  r>  r7  r8  s         rN   prune_intermediate_layers!ByobNet.prune_intermediate_layers  sq      #7s4??7KW"UOOI.	kk*9-r** kkmDO!!!R(rM   c                    U R                  U5      nU R                  (       a:  [        R                  R	                  5       (       d  [        U R                  U5      nOU R                  U5      nU R                  U5      nU$ )zcForward pass through feature extraction.

Args:
    x: Input tensor.

Returns:
    Feature tensor.
)r  r  r   r4  r5  r)   r  r  r   s     rN   forward_featuresByobNet.forward_features  s\     IIaL""599+A+A+C+Ct{{A.AAAOOArM   
pre_logitsc                 P    U(       a  U R                  XS9$ U R                  U5      $ )zForward pass through head.

Args:
    x: Input features.
    pre_logits: Return features before final linear layer.

Returns:
    Classification logits or features.
)rE  )r  )r   r   rE  s      rN   forward_headByobNet.forward_head  s&     7Atyyy2RdiiPQlRrM   c                 J    U R                  U5      nU R                  U5      nU$ )zGForward pass.

Args:
    x: Input tensor.

Returns:
    Output logits.
)rC  rG  r   s     rN   r   ByobNet.forward  s)     !!!$a rM   )r  r  r  r  r  r  rd   r  r  r[   r  r  r  )  r   NrX   Nr   r   r   r   TNNr  Trh   )NFFr3  FF)r%   FT)"rA   rB   rC   rD   rE   r-   rJ   r
   rF   r   r   rK   rp   r   r   r4  ignorer	   r   r  r"  rG   rH   r&  r*  r   r   r  r@  rC  rG  r   rL   r   r   s   @rN   r,   r,     s     $)-!#>B!%'#$$&#'SQSQ SQ 	SQ
 "#SQ SQ uS%S/%9:;SQ SQ #SQ !SQ "SQ !SQ SQj YYD T#s(^  $ YY)T )T ) ) YY		  2C 2hsm 2W[ 2 8<$$',',< ||<  eCcN34<  	< 
 <  <  !%<  !%<  
tELL!5tELL7I)I#JJ	K< @ ./$#	3S	>*  	
 
c2%,, 5<< "
Sell 
S 
S 
S %,,  rM   r,   r  rH  r\   c                    [        U [        R                  5      (       a  U R                  S   U R                  S   -  U R                  -  nX0R
                  -  nU R                  R                  R                  S[        R                  " SU-  5      5        U R                  b%  U R                  R                  R                  5         gg[        U [        R                  5      (       aa  [        R                  R                  U R                  SSS9  U R                  b*  [        R                  R                  U R                  5        gg[        U [        R                   5      (       aS  [        R                  R#                  U R                  5        [        R                  R                  U R                  5        g[%        U S5      (       a  U R'                  US	9  gg)
zInitialize weights.

Args:
    module: Module to initialize.
    name: Module name.
    zero_init_last: Zero-initialize last layer.
r   r%          @Nr   g{Gz?)meanrk  r   r  )r   rG   r-  r   r+  rr   r   rB  r7  mathrc  r,  zero_Linearr   r   r6  ones_r   r   )r  rH  r\   fan_outs       rN   r  r     sB    &"))$$$$Q'&*<*<Q*??&BUBUUMM!""1diig&>?;;"KK""$ #	FBII	&	&
CT:;;"GGNN6;;' #	FBNN	+	+
fmm$
v{{#		(	(>: 
)rM   gernet_lr  rw   r9   )r3   r4   r5   r7   r8   r:      r     i  r           @rX   i 
  )rQ   rY   rW   r[   gernet_mgernet_s0   r    i0  rx      i  	repvgg_a0)r6   r     r%   )      ?rb  rb        @)r4   rq   r~   )rQ   rU   rY   	repvgg_a1)r%   r%   r%   rc  rv   	repvgg_a2)      ?rf  rf  g      @	repvgg_b0)r9   r9   r9   rc  )rq   	repvgg_b1)rO  rO  rO        @repvgg_b1g4)rq   rr   	repvgg_b2)rc  rc  rc        @repvgg_b2g4	repvgg_b3)rZ  rZ  rZ  rl  repvgg_b3g4repvgg_d2se)r  ra     r%   r   g      ?)rd_ratio
rd_divisor)rQ   rU   rY   r;   r<   	resnet51qry   i   r  i   silu)rQ   rY   rU   rW   r[   r_   	resnet61qr  )r3   r4   r5   r7   r8   r:   r?   r  )r  )rQ   rY   rU   rW   r[   r_   r?   resnext26tsi   r  rV   )rQ   rY   rU   rW   r_   gcresnext26tsgca)rQ   rY   rU   rW   r_   r;   seresnext26tseca_resnext26tsecabat_resnext26tsbatr  )r  )rQ   rY   rU   rW   r_   r;   r<   
resnet32ts
resnet33tsi   gcresnet33ts)rQ   rY   rU   rW   r[   r_   r;   seresnet33tseca_resnet33tsgcresnet50t)r3   r4   r5   r7   r:   )rQ   rY   rU   rW   r;   gcresnext50tsregnetz_b16   `         )rr  )r  r   )	rQ   rY   rW   rS   r[   r_   r;   r<   r?   regnetz_c16regnetz_d32i   )
rQ   rY   rU   rW   rS   r[   r_   r;   r<   r?   
regnetz_d8
regnetz_e8regnetz_b16_evos)r   )
rQ   rY   rW   rS   r[   r_   ra   r;   r<   r?   regnetz_c16_evosregnetz_d8_evosr  )rQ   rY   rU   rW   rS   r[   r_   ra   r;   r<   r?   mobileone_s0)rb  r9   r9   rO  )rq   r   r   mobileone_s1)rf  rf  rO  rc  mobileone_s2)rf  rO  rc  ri  mobileone_s3)rO  rc  rZ  ri  mobileone_s4)rZ        @r  ri  )r   r   rY  r%   )rq   r   resnet50_clip)rX   rX   rv   r  r   r	  )rQ   rY   rU   rW   rS   rc   rf   resnet101_clip   resnet50x4_clip
   g      ?)rQ   rZ   rY   rU   rW   rS   rc   rf   resnet50x16_clip   rf  resnet50x64_clip   $   rO  resnet50_mlpr  )rQ   rY   rU   rW   rS   rc   rd   rf   test_byobnetr  r  rq  r^   )rQ   rY   rS   rW   r_   r;   r<   )r  r  r  r  r  re   )rf   _gap
state_dictmodelprefixc                    [        UR                  [        [        45      nSS KnS nS n0 nU R                  5        GH	  u  pUR                  U5      (       d  M  UR                  U S3SU5      nUR                  U S3SU5      nUR                  U S3XX5      nUR                  U S	3Xh5      nUR                  U S
35      (       ax  U(       d  M  UR                  US
-   S5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nXU'   GM     U$ )Nr   c                     [        U R                  S5      5      S-
  n[        U R                  S5      5      U R                  S5      [        U R                  S5      5      pCnSU SU S3nSSS	S
.nXd   U-   nXW-   $ )Nr%   r6   r   r  r  r  z
conv1_1x1.z
conv2_kxk.z
conv3_1x1.)r%   r6   r   rJ   group)r8  r  	layer_idx
layer_typelayer_id
prefix_strid_map
suffix_strs           rN   
_stage_sub(_convert_openai_clip.<locals>._stage_sub^	  sz    
Oa'	*-aggaj/1771:s177ST:x	yk9+Q7
!l|D%
2
&&rM   c                     [        U R                  S5      5      S-
  n[        U R                  S5      5      [        U R                  S5      5      p2SU SU S3US:X  a  S-   $ S	-   $ )
Nr%   r6   r   r  r  z
.shortcut.r   z	conv.convzconv.bnr  )r8  r  r  r  s       rN   	_down_sub'_convert_openai_clip.<locals>._down_subf	  sa    
Oa'	!!''!*os1771:81YKz:XYZ]kjj`ijjrM   zconv([0-9])zstem.conv\1.convz	bn([0-9])zstem.conv\1.bnz'layer([0-9])\.([0-9]+)\.([a-z]+)([0-9])z+layer([0-9])\.([0-9]+)\.downsample\.([0-9])attnpoolr  positional_embedding	pos_embedq_projqk_projkv_projvc_projproj)	r   r  r   r   reitems
startswithsubr   )
r  r  r  model_has_attn_poolr  r  r  out_dictr  r  s
             rN   _convert_openai_clipr  V	  sK   
 %UZZ2Do1VW'k
 H  "||F##FFvhk*,?CFFvhi(*;Q?FFvhEF
VFFvhIJIY<<6((+,,&		&:-v6A		0+>A		(C(A		(C(A		(C(A		(F+A! #$ OrM   c                 (    SU ;   a  [        X5      n U $ )Nzvisual.conv1.weight)r  )r  r  s     rN   checkpoint_filter_fnr  	  s     
*)*<
rM   variant
pretrainedc           	      P    [        [        X4[        U    [        [	        SS9S.UD6$ )zCreate a ByobNet model.

Args:
    variant: Model variant name.
    pretrained: Load pretrained weights.
    **kwargs: Additional model arguments.

Returns:
    ByobNet model instance.
T)flatten_sequential)r  pretrained_filter_fnfeature_cfg)r&   r,   
model_cfgsr  rj   )r  r  r   s      rN   _create_byobnetr  	  s8      W%1D1	
  rM   urlc                 2    U SSSSS[         [        SSSS	.UE$ )
zCreate default configuration dictionary.

Args:
    url: Model weight URL.
    **kwargs: Additional configuration options.

Returns:
    Configuration dictionary.
rK  r      r  r  r        ?bilinear	stem.convhead.fc
apache-2.0r  r  
input_size	pool_sizecrop_pctinterpolationrP  rk  
first_convre   licenser   r   r  r   s     rN   _cfgr  	  s5     4}SYJ%.B!  rM   c                 2    U SSSSS[         [        SSSS	.UE$ )
zCreate RepVGG configuration dictionary.

Args:
    url: Model weight URL.
    **kwargs: Additional configuration options.

Returns:
    Configuration dictionary.
rK  r   rx   rx   r  r  ?bicubiczstem.conv1.convr  r  r  r  r  s     rN   _cfgrr  	  s5     4}SY)%.B'y  rM   zgernet_s.idstcv_in1kztimm/)	hf_hub_idzgernet_m.idstcv_in1kzgernet_l.idstcv_in1kr  r  )r  r  r  zrepvgg_a0.rvgg_in1k)zstem.conv_kxk.convzstem.conv_1x1.convmit)r  r  r  zrepvgg_a1.rvgg_in1kzrepvgg_a2.rvgg_in1kzrepvgg_b0.rvgg_in1kzrepvgg_b1.rvgg_in1kzrepvgg_b1g4.rvgg_in1kzrepvgg_b2.rvgg_in1kzrepvgg_b2g4.rvgg_in1kzrepvgg_b3.rvgg_in1kzrepvgg_b3g4.rvgg_in1kzrepvgg_d2se.rvgg_in1k)r   @  r  )r  r  )r  r  r  r  r  r  zresnet51q.ra2_in1kzkhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet51q_ra2-d47dcc76.pthz
stem.conv1)r   r  r  )r  r  r  r  r  test_input_sizetest_crop_pctzresnet61q.ra2_in1kzkhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet61q_ra2-6afc536c.pth)r  r  r  r  zresnext26ts.ra2_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnext26ts_256_ra2-8bbd9106.pthzseresnext26ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/seresnext26ts_256-6f0d74a3.pthzgcresnext26ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnext26ts_256-e414378b.pthzeca_resnext26ts.ch_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/eca_resnext26ts_256-5a1d030f.pthzbat_resnext26ts.ch_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/bat_resnext26ts_256-fa6fd595.pth)r  r  min_input_sizezresnet32ts.ra2_in1kzqhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnet32ts_256-aacf5250.pthzresnet33ts.ra2_in1kzqhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnet33ts_256-e91b09a4.pthzgcresnet33ts.ra2_in1kzshttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnet33ts_256-0e0cd345.pthzseresnet33ts.ra2_in1kzshttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/seresnet33ts_256-f8ad44d9.pthzeca_resnet33ts.ra2_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/eca_resnet33ts_256-8f98face.pthzgcresnet50t.ra2_in1kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnet50t_256-96374d1c.pthzgcresnext50ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnext50ts_256-3e0f515e.pthzregnetz_b16.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_b_raa-677d9606.pthr  )r  r  r  r  r  gGz?)
r  r  r  rP  rk  r  r  r  r  r  zregnetz_c16.ra3_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_c_rab2_256-a54bf36a.pth)r  r  r  rP  rk  r  r  r  zregnetz_d32.ra3_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_d_rab_256-b8073a89.pthgffffff?)r  r  rP  rk  r  r  zregnetz_d8.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_d8_bh-afc03c55.pth)r  r  rP  rk  r  r  r  zregnetz_e8.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_e8_bh-aace8e6e.pthzregnetz_b16_evos.untrained)r  rP  rk  r  r  r  r  zregnetz_c16_evos.ch_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/regnetz_c16_evos_ch-d8311942.pth)r  r  r  rP  rk  r  r  zregnetz_d8_evos.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/regnetz_d8_evos_ch-2bc12646.pthzmobileone_s0.apple_in1kr  )zstem.conv_kxk.0.convzstem.conv_scale.convzmobileone-license)r  r  r  r  zmobileone_s1.apple_in1kr  zmobileone_s2.apple_in1kzmobileone_s3.apple_in1kzmobileone_s4.apple_in1kzresnet50_clip.openaiz	head.proj)	r  r  rP  rk  r]   r  r  re   r  zresnet101_clip.openaizresnet50x4_clip.openai)	   r  zresnet50x16_clip.openaii   )r   r^  r^  )r  r  zresnet50x64_clip.openai)r     r  )ra  ra  zresnet50_clip.cc12mzresnet50_clip.yfcc15mzresnet101_clip.yfcc15mzresnet50_clip_gap.openai)r  r  rP  rk  r  r  r  zresnet101_clip_gap.openai)r  r  )r      r  )rY  rY  )r  r  r  r  r  )zresnet50x4_clip_gap.openaizresnet50x16_clip_gap.openaizresnet50x64_clip_gap.openaizresnet50_clip_gap.cc12mzresnet50_clip_gap.yfcc15mzresnet101_clip_gap.yfcc15mzresnet50_mlp.untrainedztest_byobnet.r160_in1kc                     [        SSU 0UD6$ )zGEResNet-Large (GENet-Large from official impl)
`Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
r  )rV  r  r  r   s     rN   rV  rV  
      
 G*GGGrM   c                     [        SSU 0UD6$ )zGEResNet-Medium (GENet-Normal from official impl)
`Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
r  )r[  r  r  s     rN   r[  r[  
  r  rM   c                     [        SSU 0UD6$ )zEResNet-Small (GENet-Small from official impl)
`Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
r  )r\  r  r  s     rN   r\  r\  
  r  rM   c                     [        SSU 0UD6$ )zURepVGG-A0
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )r`  r  r  s     rN   r`  r`        
 H:HHHrM   c                     [        SSU 0UD6$ )zURepVGG-A1
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rd  r  r  s     rN   rd  rd    r  rM   c                     [        SSU 0UD6$ )zURepVGG-A2
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )re  r  r  s     rN   re  re    r  rM   c                     [        SSU 0UD6$ )zURepVGG-B0
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rg  r  r  s     rN   rg  rg    r  rM   c                     [        SSU 0UD6$ )zURepVGG-B1
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rh  r  r  s     rN   rh  rh  &  r  rM   c                     [        SSU 0UD6$ )zWRepVGG-B1g4
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rj  r  r  s     rN   rj  rj  .      
 JZJ6JJrM   c                     [        SSU 0UD6$ )zURepVGG-B2
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rk  r  r  s     rN   rk  rk  6  r  rM   c                     [        SSU 0UD6$ )zWRepVGG-B2g4
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rm  r  r  s     rN   rm  rm  >  r  rM   c                     [        SSU 0UD6$ )zURepVGG-B3
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rn  r  r  s     rN   rn  rn  F  r  rM   c                     [        SSU 0UD6$ )zWRepVGG-B3g4
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )ro  r  r  s     rN   ro  ro  N  r  rM   c                     [        SSU 0UD6$ )zWRepVGG-D2se
`Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
r  )rp  r  r  s     rN   rp  rp  V  r  rM   c                     [        SSU 0UD6$ )
    r  )rt  r  r  s     rN   rt  rt  ^       H:HHHrM   c                     [        SSU 0UD6$ )r
  r  )rv  r  r  s     rN   rv  rv  e  r  rM   c                     [        SSU 0UD6$ )r
  r  )rw  r  r  s     rN   rw  rw  l       JZJ6JJrM   c                     [        SSU 0UD6$ )r
  r  )rx  r  r  s     rN   rx  rx  s       LzLVLLrM   c                     [        SSU 0UD6$ )r
  r  )rz  r  r  s     rN   rz  rz  z  r  rM   c                     [        SSU 0UD6$ )r
  r  )r{  r  r  s     rN   r{  r{         NNvNNrM   c                     [        SSU 0UD6$ )r
  r  )r}  r  r  s     rN   r}  r}    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r         IJI&IIrM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r         KjKFKKrM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r         M
MfMMrM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r         O*OOOrM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r%  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )z/OpenAI Modified ResNet-50 CLIP image tower
    r  )r  r  r  s     rN   r  r    r  rM   c                     [        SSU 0UD6$ )z0OpenAI Modified ResNet-101 CLIP image tower
    r  )r  r  r  s     rN   r  r  "  r  rM   c                     [        SSU 0UD6$ )z1OpenAI Modified ResNet-50x4 CLIP image tower
    r  )r  r  r  s     rN   r  r  )  r  rM   c                     [        SSU 0UD6$ )z2OpenAI Modified ResNet-50x16 CLIP image tower
    r  )r  r  r  s     rN   r  r  0  r%  rM   c                     [        SSU 0UD6$ )z2OpenAI Modified ResNet-50x64 CLIP image tower
    r  )r  r  r  s     rN   r  r  7  r%  rM   c                     [        SSU 0UD6$ )zOOpenAI Modified ResNet-50 CLIP image tower w/ avg pool (no attention pool)
    r  )resnet50_clip_gapr  r  s     rN   r3  r3  >  s     P:PPPrM   c                     [        SSU 0UD6$ )zPOpenAI Modified ResNet-101 CLIP image tower w/ avg pool (no attention pool)
    r  )resnet101_clip_gapr  r  s     rN   r5  r5  E  s     QJQ&QQrM   c                     [        SSU 0UD6$ )zQOpenAI Modified ResNet-50x4 CLIP image tower w/ avg pool (no attention pool)
    r  )resnet50x4_clip_gapr  r  s     rN   r7  r7  L  s     RZR6RRrM   c                     [        SSU 0UD6$ )zROpenAI Modified ResNet-50x16 CLIP image tower w/ avg pool (no attention pool)
    r  )resnet50x16_clip_gapr  r  s     rN   r9  r9  S       SjSFSSrM   c                     [        SSU 0UD6$ )zROpenAI Modified ResNet-50x64 CLIP image tower w/ avg pool (no attention pool)
    r  )resnet50x64_clip_gapr  r  s     rN   r<  r<  Z  r:  rM   c                     [        SSU 0UD6$ )r
  r  )r  r  r  s     rN   r  r  a  r  rM   c                     [        SSU 0UD6$ )z,Minimal test ResNet (BYOB based) model.
    r  )r  r  r  s     rN   r  r  h  r  rM   ))r  rX  r  r%   r9   r9   r9   r9   r   ))r6   r  r  r%   r?  r@   r%   )r%   F)rb   rb   r  NNN)r6   )r   r   r  rL  )rb   F)zvisual.r  )rb   )rE   rQ  dataclassesr   r   r   	functoolsr   typingr   r   r	   r
   r   r   r   r   r   r   torch.nnrG   	timm.datar   r   r   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   _builderr&   	_featuresr'   _manipulater(   r)   	_registryr*   r+   __all__r.   r-   rJ   rK   r   r   rF   rp   r   r   r   r   rH   r   r   r   r  r  r  r&  ro  r  rj   r  r  r0   r  r  r/   r  r  r  r  r  r  r,   r  r  r  r   r  r  r  r  r  default_cfgsrV  r[  r\  r`  rd  re  rg  rh  rj  rk  rm  rn  ro  rp  rt  rv  rw  rx  rz  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r3  r5  r7  r9  r<  r  r  r@   rM   rN   <module>rL     s$  <  1 1  T T T   d d    & + + 4 <
W 2 2 2( I I IBU38_ %s
:K hk  k3 * + 0%'!"	"c?"%*" c?" 	"
 
${
"P ()	S#X S$s)^$ 	 ;@k8K;P.P(Q VZ[fVg "&8C= &C &C &$ 0 0 0/'BII /'d      	 
 S/    bii FG GTWbii WtO		 OdL		 L^s=")) s=lK=RYY K=\QBII Qh 
		+s +bii +,c299n- ,N2== Nh !/// / 	/
 / /dR	!Yd38n !Y !YYd !YJ   
(7
	D "$ #'$(.AE;E;E; E; S>	E;
 E; E; C=E; !E; "(+E;P
{ 
d 
Lbii L^
;")) ;3 ;T ;VZ ;2  xWSA!CWSA!CXcQ1GXcQ1DXcQ1D
 x WSA!CWSA!CXcQ1GXcQ1DXcQ1D
 x2 WR1rBWR1rBXcQ1GXcQ1DXcQ1D
 3xL }1HIMxV }@Wx` }1FGaxj  12kxt  01ux~  0;xH  34IxR  3A>Sx\  01]xf  0;gxp ~2EF&Q7qxD XcQ2$GXcQ2$GXdaB4HXdaA#F	
 Exd VqC1sQUQWXXcQ2$GXdaB4HXdaA#F	
 T*exF XcQ2$GXcQ2$GXdaB4HXdaB4H	
 Gx^ XcQ2$GXcQ2$GXdaB4HXdaB4H	
 _xx XcQ2$GXcQ2$GXdaB4HXdaB4H	
 yxR  XcQ2$GXcQ2$GXdaB4HXdaB4H	
 Sxl  XcQ2$GXcQ2$GXdaB4HXdaB4H	
 A&mxL XcQ1FXcQ1FXdaA$GXdaA$G	
 Mxj XcQ1FXcQ1FXdaA$GXdaA$G	
 kxJ XcQ1FXcQ1FXdaA$GXdaA$G	
 Kxf XcQ1FXcQ1FXdaA$GXdaA$G	
 gxB XcQ1FXcQ1FXdaA$GXdaA$G	
 Cx` XcQ4@XcQ4@XdaDAXdaDA	
 axz XcQ2$GXcQ2$GXdaB4HXdaB4H	
 {xX	 XbA"CXbA"CXsaB1EXcQ2!D	
 $'DT:Y	xx	 XbA"CXbA"CXsaB1EXcQ2!D	
 $'DT:y	xX
 XbA"CXcQ2!DXsaB1EXcQ2!D	
 $'DT:Y
xz
 XbA!BXcQ1CXsaA!DXcQ1C	
 $'DT:{
x\ XbA!BXcQ1CXsaA!DXcQ1C	
 $'DT:]xB !XbA"CXbA"CXsaB1EXcQ2!D	
 <B7$'DT:Cxd !XbA"CXbA"CXsaB1EXcQ2!D	
 <B7$'DT:exF  XbA!BXcQ1CXsaA!DXcQ1C	
 <B7$'DT:!Gxl "6!Lmxv "67wx@ "67AxJ "67KxT "6,OUx` XcQ4@XcQ4@XdaDAXdaDA	
 ax| XcQ4@XcQ4@XtqTBXdaDA	
 }xX  XcQ4@XcQ4@XtqTBXdaDA	
 Yxv !XcQ4@XcQ4@XtqTBXdaDA	
 wxT !XcQ4@XsaDAXtqTBXtqTB	
 Uxt XcQ4@XcQ4@XdaDAXdaDA	
 uxT VqB!cBVqB!cBWSA"FXcQ2$G	
 $'Ux
r 
hA$Z]lKJq6z 
h  (ell*+(( ( 
#u||
	(Vell*+S d  (c T#s(^ (s d38n ( % e&D73e& D73e& D7}X^_	e& 4?Pe& 4?Pe& 4?Pe&  4?P!e&& 4?P'e&, T?P-e&2 4?P3e&8 T?P9e&> 4?P?e&D T?PEe&J T? HsKe&X $yMV%S	:Ye&b %y%S:ce&n E E%S:oe&v U C%S:we&~ U C%S:e&F u E%S :Ge&N u E$ &Oe&Z 5%S:[e&b 5%S:ce&j U B%S:ke&r U B%S:se&z u D%S :{e&D E A%S:Ee&N U C%S:Oe&Z E~_/ FTS`ps	u[e&d E D_/}C	Iee&n E C/DR_aoe&v 5~/DR_ortwe&~ 5~/DR_orte&H !%_/ FTS`#bIe&N  D_/}	!6Oe&X u C/DR_or tYe&b tC#	 ce&n tC#	 oe&z tC#	 {e&F tC#	 Ge&R tC#	 Se&b E/_-6ce&p U.O-6qe&~ e.O-6e&L u.O-8 Me&Z u/_-8 [e&h 5/_-6ie&v U/_-6we&D e.O-6Ee&V ,/ F	!We&b  ,/ F	"ce&n #(,/ F	# $),/ H	$ $),/ H	$  %,/ F	  "',/ F	" #(,/ F	# $ F $ 46A	e& eP	 HG H H HG H H HG H H IW I I IW I I IW I I IW I I IW I I Kw K K IW I I Kw K K IW I I Kw K K Kw K K IW I I IW I I Kw K K M M M M M M O7 O O O7 O O Jg J J Jg J J L L L L L L N' N N Kw K K M M M Kw K K Kw K K Kw K K Jg J J Jg J J PG P P PG P P O7 O O L L L L L L L L L L L L L L L M M M N' N N O7 O O PG P P PG P P QW Q Q Rg R R Sw S S T T T T T T L L L L L LrM   