
    RЦi                     2   S r SSKJr  SSKJrJr  SSKJr  SSKJ	r	J
r
JrJrJr  SSKrSSKJr  SSKJrJr  SSKJrJrJrJrJrJrJrJrJrJr  S	S
KJ r   S	SK!J"r"  S	SK#J$r$  S	SK%J&r&J'r'  SS/r(\ " S S5      5       r) " S S\RT                  5      r+GSS\,S\-S\
4S jjr. " S S\RT                  5      r/\" " S S\RT                  5      5       r0      GSS\1S\1S\,S \\
   S!\\
   S"\2S\\Rf                  \1\\,\	4   4   4S# jjr4\5" SS$S%S&S'S(S)S*S+S,S-S.S/S0S1S29r6 " S3 S\RT                  5      r7     GSS4\\1S54   S6\\1S54   S7\\1   S!\,S8\\,   S9\\\,\	4      S\)4S: jjr8GSS4\\1S54   S6\\1S54   S\)4S; jjr9       GSS4\\1S54   S6\\1S54   S7\1S=\-S>\-S!\,S8\,S9\\\,\	4      S\)4S? jjr:   GS	S4\\1S54   S6\\1S54   S!\,S@\2S\)4
SA jjr;\5" GS
0 SB\;" SCSD9_SE\;" SFSD9_SG\;" SHSD9_SI\;" SJSD9_SK\;" SLSD9_SM\;" SNSD9_SO\;" SPSD9_SQ\:" SCSD9_SR\:" SFSD9_SS\:" SHSD9_ST\:" SJSD9_SU\:" SLSD9_SV\:" SNSD9_SW\:" SPSD9_SX\:" SYSD9_SZ\:" SCS[S\S]\5" S]S^S_9S`Sa9_Sb\:" SCS[S\S]Sc\5" 5       S`Sd9_Se\:" SFSfS\S]Sc\5" 5       S`Sd9_Sg\:" SHSfS\S]Sc\5" 5       S`Sd9_Sh\:" SJSfS\S]Sc\5" 5       S`Sd9_Si\9" SjSD9_Sk\9" SlSD9_Sm\9" SnSoSp9_Sq\9" SrSsSp9_St\9" SuSvSp9_Sw\9" SxSySp9_Sz\8" S{SD9_S|\8" S}SD9_S~\8" SSD9_S\8" S{S<\5" SS9S9_S\8" S}S<\5" SS9S9_S\8" SS<\5" SS9S9_S\8" S{Sc\5" 5       S9_S\8" S}Sc\5" 5       S9_S\8" SSc\5" 5       S9_S\:" SSS[S^S]\5" S]S^S_9S`S9_6r<GSS\,S\2S\	S\74S jjr=GSS\,S\	S\\,\	4   4S jjr>\&" 0 S\>" SSSSSSSS9_S\>" SSSSSSSS9_S\>" SSSSSSSS9_S\>" SSSSSSSS9_S\>" SSSSSSSS9_S\>" SSSSSSSS9_S\>" SSSSSSSS9_SQ\>" SSSSS9_SR\>" SSSSS9_SS\>" SSSSS9_ST\>" SSSSS9_SU\>" SSSSS9_SV\>" SSSSS9_SW\>" SSSSS9_SX\>" SSSSS9_S\>" SSSSSSS9_S\>" SSSSSSS9_0 S\>" SSSSSSS9_S\>" SSSSSSS9_Sh\>" SSSSSS9_Si\>" SSSSSS9_S\>" SSSSSSS9_Sm\>" SSSSSS9_Sq\>" SSSSSS9_St\>" SSSSSS9_Sw\>" SSSSSS9_Sz\>" SSS9_S\>" SSSSSSSS9_S~\>" SSS9_S\>" SSS9_S\>" SSS9_S\>" SSS9_S\>" SSS9_S\>" SSS9_E\>" SSS9\>" SSSSSSS9S.E5      r?\'GSS\2S\	S\74S jj5       r@\'GSS\2S\	S\74S jj5       rA\'GSS\2S\	S\74S jj5       rB\'GSS\2S\	S\74S jj5       rC\'GSS\2S\	S\74S jj5       rD\'GSS\2S\	S\74S jj5       rE\'GSS\2S\	S\74S jj5       rF\'GSS\2S\	S\74S jj5       rG\'GSS\2S\	S\74S jj5       rH\'GSS\2S\	S\74S jj5       rI\'GSS\2S\	S\74S jj5       rJ\'GSS\2S\	S\74S jj5       rK\'GSS\2S\	S\74S jj5       rL\'GSS\2S\	S\74S jj5       rM\'GSS\2S\	S\74S jj5       rN\'GSS\2S\	S\74S jj5       rO\'GSS\2S\	S\74S jj5       rP\'GSS\2S\	S\74S jj5       rQ\'GSS\2S\	S\74S jj5       rR\'GSS\2S\	S\74S jj5       rS\'GSS\2S\	S\74S jj5       rT\'GSS\2S\	S\74S jj5       rU\'GSS\2S\	S\74S jj5       rV\'GSS\2S\	S\74S jj5       rW\'GSS\2S\	S\74S jj5       rX\'GSS\2S\	S\74S jj5       rY\'GSS\2S\	S\74S jj5       rZ\'GSS\2S\	S\74S jj5       r[\'GSS\2S\	S\74S jj5       r\\'GSS\2S\	S\74S jj5       r]\'GSS\2S\	S\74S jj5       r^\'GSS\2S\	S\74S jj5       r_\'GSS\2S\	S\74GS  jj5       r`\'GSS\2S\	S\74GS jj5       ra\'GSS\2S\	S\74GS jj5       rb\'GSS\2S\	S\74GS jj5       rcg(  a  Normalization Free Nets. NFNet, NF-RegNet, NF-ResNet (pre-activation) Models

Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets`
    - https://arxiv.org/abs/2101.08692

Paper: `High-Performance Large-Scale Image Recognition Without Normalization`
    - https://arxiv.org/abs/2102.06171

Official Deepmind JAX code: https://github.com/deepmind/deepmind-research/tree/master/nfnets

Status:
* These models are a work in progress, experiments ongoing.
* Pretrained weights for two models so far, more to come.
* Model details updated to closer match official JAX code now that it's released
* NF-ResNet, NF-RegNet-B, and NFNet-F models supported

Hacked together by / copyright Ross Wightman, 2021.
    )OrderedDict)	dataclassreplace)partial)AnyCallableDictOptionalTupleNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
ClassifierHeadDropPathcalculate_drop_path_ratesAvgPool2dSameScaledStdConv2dScaledStdConv2dSameget_act_layer
get_act_fnget_attnmake_divisible   )build_model_with_cfg)register_notrace_module)checkpoint_seq)generate_default_cfgsregister_modelNormFreeNetNfCfgc                   v   \ rS rSr% Sr\\\\\4   \S'   \\\\\4   \S'   Sr\	\S'   Sr
\\S'   S	r\\   \S
'   S	r\\   \S'   S	r\\   \S'   S	r\\\\4      \S'   Sr\	\S'   Sr\	\S'   Sr\	\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\	\S'   Sr\\S'   Sr\\S '   S!r \\S"'   S#r!g	)$r    &   z.Configuration for Normalization-Free Networks.depthschannelsg?alpha3x3	stem_typeNstem_chs
group_size
attn_layerattn_kwargs       @	attn_gain      ?width_factor      ?bottle_ratior   num_features   ch_divFreg
extra_convgamma_in_actsame_paddinggh㈵>std_conv_epsskipinitzero_init_fcsilu	act_layer )"__name__
__module____qualname____firstlineno____doc__r   int__annotations__r%   floatr'   strr(   r
   r)   r*   r+   r	   r   r-   r/   r1   r2   r4   r5   boolr6   r7   r8   r9   r:   r;   r=   __static_attributes__r>       P/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/models/nfnet.pyr    r    &   s   8#sC$%%Cc3&''E5Is"Hhsm" $J$ $J$,0K$sCx.)0IuL%L%L#FCOCJL$L$L%HdL$IsrJ   c                   z   ^  \ rS rSrSrSS\S\S\4U 4S jjjrS\	R                  S\	R                  4S	 jrS
rU =r$ )GammaAct@   z.Activation function with gamma scaling factor.act_typegammainplacec                 Z   > [         TU ]  5         [        U5      U l        X l        X0l        g)zInitialize GammaAct.

Args:
    act_type: Type of activation function.
    gamma: Scaling factor for activation output.
    inplace: Whether to perform activation in-place.
N)super__init__r   act_fnrP   rQ   )selfrO   rP   rQ   	__class__s       rK   rT   GammaAct.__init__C   s&     	 *
rJ   xreturnc                 f    U R                  XR                  S9R                  U R                  5      $ )zRForward pass.

Args:
    x: Input tensor.

Returns:
    Scaled activation output.
rQ   )rU   rQ   mul_rP   rV   rY   s     rK   forwardGammaAct.forwardP   s(     {{1ll{388DDrJ   )rU   rP   rQ   )relur.   F)r?   r@   rA   rB   rC   rG   rF   rH   rT   torchTensorr_   rI   __classcell__rW   s   @rK   rM   rM   @   sI    8 e D  	E 	E%,, 	E 	ErJ   rM   r.   rO   rP   rZ   c                 8   ^ ^ SS[         S[        4U U4S jjjnU$ )zCreate activation function factory with gamma scaling.

Args:
    act_type: Type of activation function.
    gamma: Scaling factor for activation output.

Returns:
    Activation function factory.
rQ   rZ   c                    > [        TTU S9$ )N)rP   rQ   )rM   )rQ   rO   rP   s    rK   _createact_with_gamma.<locals>._createf   s    w??rJ   F)rH   rM   )rO   rP   rh   s   `` rK   act_with_gammark   \   s$    @ @( @ @NrJ   c                      ^  \ rS rSrSrSSS\SS4S\S\S\S\S	\\   S
\4U 4S jjjr	S\
R                  S\
R                  4S jrSrU =r$ )DownsampleAvgk   zEAvgPool downsampling as in 'D' ResNet variants with dilation support.r   Nin_chsout_chsstridedilationfirst_dilation
conv_layerc	           	         > [         TU ]  5         US:X  a  UOSn	US:  d  US:  a1  U	S:X  a  US:  a  [        O[        R                  n
U
" SU	SSS9U l        O[        R                  " 5       U l        U" XSSXxS9U l        g)zInitialize DownsampleAvg.

Args:
    in_chs: Input channels.
    out_chs: Output channels.
    stride: Stride for downsampling.
    dilation: Dilation rate.
    first_dilation: First dilation rate (unused).
    conv_layer: Convolution layer type.
r      TF)	ceil_modecount_include_pad)rq   devicedtypeN)rS   rT   r   nn	AvgPool2dpoolIdentityconv)rV   ro   rp   rq   rr   rs   rt   ry   rz   
avg_strideavg_pool_fnrW   s              rK   rT   DownsampleAvg.__init__n   sv    * 	'1}V!
A:A+5?x!|-QSQ]Q]K#AzTUZ[DIDIv!FX	rJ   rY   rZ   c                 B    U R                  U R                  U5      5      $ )zLForward pass.

Args:
    x: Input tensor.

Returns:
    Downsampled tensor.
r   r}   r^   s     rK   r_   DownsampleAvg.forward   s     yy1&&rJ   r   )r?   r@   rA   rB   rC   r   rD   r
   r   rT   rb   rc   r_   rI   rd   re   s   @rK   rm   rm   k   s    O ,0#2YY Y 	Y
 Y %SMY !Y Y<	' 	'%,, 	' 	'rJ   rm   c            %          ^  \ rS rSrSrSSSSSSSSSSSSSS	S\S
SS4S\S\\   S\S\S\\   S\S\S\S\\   S\S\	S\	S\	S\\
   S\S\\
   S\
S\4$U 4S jjjrS\R                  S\R                  4S  jrS!rU =r$ )"NormFreeBlock   z-Normalization-Free pre-activation block.
    Nr   r.         ?TFr,           ro   rp   rq   rr   rs   r%   betar1   r)   r4   r5   r6   r:   r*   r-   r=   rt   drop_path_ratec                   > UUS.n[         TU ]  5         U=(       d    UnU=(       d    Un[        U(       a  X-  OX(-  U
5      nU	(       d  SOUU	-  nU	(       a  X-  S:X  a  U	U-  nX`l        Xpl        Xl        X:w  d  US:w  d  XE:w  a  [        UU4UUUUS.UD6U l        OSU l        U" 5       U l        U" UUS40 UD6U l	        U" SS9U l
        U" UUS4X5US	.UD6U l        U(       a   U" SS9U l        U" UUS4SUUS	.UD6U l        OSU l        SU l        U(       a  Ub  U" U40 UD6U l        OSU l        U" 5       U l        U" UUS4S
U(       a  SOS0UD6U l        U(       d  Ub  U" U40 UD6U l        OSU l        US:  a  [%        U5      O[&        R(                  " 5       U l        U(       a0  [&        R,                  " [.        R0                  " S0 UD65      U l        gSU l        g)a  Initialize NormFreeBlock.

Args:
    in_chs: Input channels.
    out_chs: Output channels.
    stride: Stride for convolution.
    dilation: Dilation rate.
    first_dilation: First dilation rate.
    alpha: Alpha scaling factor for residual.
    beta: Beta scaling factor for pre-activation.
    bottle_ratio: Bottleneck ratio.
    group_size: Group convolution size.
    ch_div: Channel divisor for rounding.
    reg: Use RegNet-style configuration.
    extra_conv: Add extra 3x3 convolution.
    skipinit: Use skipinit initialization.
    attn_layer: Attention layer type.
    attn_gain: Attention gain factor.
    act_layer: Activation layer type.
    conv_layer: Convolution layer type.
    drop_path_rate: Stochastic depth drop rate.
ry   rz   r   r   )rq   rr   rs   rt   NTr\      )rq   rr   groups	gain_initr.   r   )r   )rS   rT   r   r%   r   r-   rm   
downsampleact1conv1act2conv2act2bconv2battnact3conv3	attn_lastr   r{   r~   	drop_path	Parameterrb   tensorskipinit_gain)rV   ro   rp   rq   rr   rs   r%   r   r1   r)   r4   r5   r6   r:   r*   r-   r=   rt   r   ry   rz   ddmid_chsr   rW   s                           rK   rT   NormFreeBlock.__init__   s   Z /'38#V #!67CY[ab$'Z*?*-2 6)G
	"!x/I+ !-% DO #DOK	9b9
d+	!qFdjqnpq
"40DJ$WgqkX^dkhjkDKDJDK:)"71b1DIDIK	!\XrSU\Y[\
z-'626DN!DN5Ca5G.1R[[]EMR\\%,,*@R*@ASWrJ   rY   rZ   c                    U R                  U5      U R                  -  nUnU R                  b  U R                  U5      nU R                  U5      nU R	                  U R                  U5      5      nU R                  b   U R                  U R                  U5      5      nU R                  b  U R                  U R                  U5      -  nU R                  U R                  U5      5      nU R                  b  U R                  U R                  U5      -  nU R                  U5      nU R                  b  UR                  U R                  5        X R                   -  U-   nU$ )zGForward pass.

Args:
    x: Input tensor.

Returns:
    Output tensor.
)r   r   r   r   r   r   r   r   r   r-   r   r   r   r   r   r]   r%   )rV   rY   outshortcuts       rK   r_   NormFreeBlock.forward   s    iilTYY& ??&s+H jjojj3(;;"++djjo.C99 ..499S>1Cjj3(>>%..4>>##66CnnS!)HHT''(JJ)
rJ   )r   r   r   r   r%   r   r-   r   r   r   r   r   r   r   r   r   )r?   r@   rA   rB   rC   r   rD   r
   rF   rH   r   rT   rb   rc   r_   rI   rd   re   s   @rK   r   r      s^    &*,0"&(,$"-1",0#2$&+\X\X c]\X 	\X
 \X %SM\X \X \X  \X !\X \X \X \X \X !*\X  !\X"  )#\X$ !%\X& "'\X \X| %,,  rJ   r    ro   rp   r'   rt   r=   preact_featurec                    XgS.nSn	[        USSS9n
[        5       nUS;   d   eSU;   a  SU;   a*  SU;  d   eUS	-  US
-  US-  U4nSnS
n	[        US-  SSS9n
O0SU;   a  SU-  S	-  US-  U4nOUS-  US-  U4nSn[        US-  SSS9n
[        U5      S-
  n[        [	        X5      5       H5  u  nu  nnU" U U4SUS.UD6USUS-    3'   X:w  a  U" SS9USUS-    3'   Un M7     O%SU;   a  U" X4SSS.UD6US'   OU" X4SSS.UD6US'   SU;   a  [
        R                  " SSSS9US'   S
n	[
        R                  " U5      X4$ )a  Create stem module for NFNet models.

Args:
    in_chs: Input channels.
    out_chs: Output channels.
    stem_type: Type of stem ('', 'deep', 'deep_tiered', 'deep_quad', '3x3', '7x7', etc.).
    conv_layer: Convolution layer type.
    act_layer: Activation layer type.
    preact_feature: Use pre-activation feature.

Returns:
    Tuple of (stem_module, stem_stride, stem_feature_info).
r   rv   	stem.convnum_chs	reductionmodule)	r   deepdeep_tiered	deep_quadr&   7x7	deep_pool3x3_pool7x7_poolr   quadr}   r3      )rv   r   r   rv   z
stem.conv3tieredr   )rv   r   r   z
stem.conv2r   )kernel_sizerq   r   Tr\   actr&      )rq   padding)dictr   len	enumeratezipr{   	MaxPool2d
Sequential)ro   rp   r'   rt   r=   r   ry   rz   r   stem_stridestem_featurestemr(   strideslast_idxicss                     rK   create_stemr     s   . 	+BK1[IL=DssssY***1glGqL'JH"GK1,WL9$K1,glGD#qL'Q,@G1,WLx=1$"3x#9:IAv1#-fa#WQq#WTV#WD4Aw }&/&=s1q5']#F	 ;
 
)	!&QqQbQV "&QqQbQV||Aa;V==99rJ   g   `U?g   yX?g   \9?g   `aK?g   ?g    ?g    `l?g   `i?g   |?g    7@g   -?g   @g   `?g   ?)identityceluelugelu
leaky_relulog_sigmoidlog_softmaxra   relu6selusigmoidr<   softsignsoftplustanhc                   (  ^  \ rS rSrSr        SS\S\S\S\S\S	\S
\S\	4U 4S jjjr
\R                  R                  SS\S\\\	4   4S jj5       r\R                  R                  SS\SS4S jj5       r\R                  R                  S\R&                  4S j5       rSS\S\\   SS4S jjrS\R.                  S\R.                  4S jrSS\R.                  S\S\R.                  4S jjrS\R.                  S\R.                  4S jrSrU =r$ )r   ip  a  Normalization-Free Network

As described in :
`Characterizing signal propagation to close the performance gap in unnormalized ResNets`
    - https://arxiv.org/abs/2101.08692
and
`High-Performance Large-Scale Image Recognition Without Normalization` - https://arxiv.org/abs/2102.06171

This model aims to cover both the NFRegNet-Bx models as detailed in the paper's code snippets and
the (preact) ResNet models described earlier in the paper.

There are a few differences:
    * channels are rounded to be divisible by 8 by default (keep tensor core kernels happy),
        this changes channel dim and param counts slightly from the paper models
    * activation correcting gamma constants are moved into the ScaledStdConv as it has less performance
        impact in PyTorch when done with the weight scaling there. This likely wasn't a concern in the JAX impl.
    * a config option `gamma_in_act` can be enabled to not apply gamma in StdConv as described above, but
        apply it in each activation. This is slightly slower, numerically different, but matches official impl.
    * skipinit is disabled by default, it seems to have a rather drastic impact on GPU memory use and throughput
        for what it is/does. Approx 8-10% throughput loss.
Ncfgnum_classesin_chansglobal_pooloutput_stride	drop_rater   kwargsc
                   > [         T"U ]  5         XS.nX l        X0l        X`l        SU l        [        U40 U
D6nUR                  [        ;   d   SUR                   S35       eUR                  (       a  [        O[        nUR                  (       a9  [        UR                  [        UR                     S9n[        XR                  S9nO;[!        UR                  5      n[        U[        UR                     UR                  S9nUR"                  (       a)  [        [%        UR"                  5      40 UR&                  D6OSn[)        UR*                  =(       d    UR,                  S	   UR.                  -  UR0                  5      n[3        UUUR4                  4UUS
.UD6u  U l        nnU/U l        [;        XqR<                  SS9nUnUnSnSn/ n[?        UR<                  5       GH  u  nnUS	:X  a  US:  a  SOSnUU:  a  US:  a  UU-  nSnUU-  nUS;   a  SOSn/ n[A        UR<                  U   5       GH&  nUS	:H  =(       a    US	:H  n[)        UR,                  U   UR.                  -  UR0                  5      nU[C        S00 SU_SU_SURD                  _SSUS-  -  _SUS	:X  a  UOS_SU_SU_SURF                  _SURH                  (       a	  U(       a  SOURJ                  _SUR0                  _SURH                  _SURL                  _SURN                  _SU_S URP                  _S!U_S"U_S#UU   U   _UD6/-  nUS	:X  a  SnUURD                  S-  -  nUnUnGM)     U =R8                  [S        UUS$U 3S%9/-  sl        U[T        RV                  " U6 /-  nGM     [T        RV                  " U6 U l,        URZ                  (       ao  [)        UR.                  URZ                  -  UR0                  5      U l-        U" UU RZ                  S40 UD6U l.        [S        U RZ                  US&S%9U R8                  S''   O!UU l-        [T        R^                  " 5       U l.        U" URZ                  S	:  S(9U l0        U RZ                  U l1        [e        U RZ                  U4UU R                  S).UD6U l3        U Ri                  5        GHO  u  n n!S*U ;   a  [k        U![T        Rl                  5      (       a  URn                  (       a*  [T        Rp                  Rs                  U!Rt                  5        O+[T        Rp                  Rw                  U!Rt                  S+S,5        U!Rx                  b+  [T        Rp                  Rs                  U!Rx                  5        M  M  [k        U![T        Rz                  5      (       d  M  [T        Rp                  R}                  U!Rt                  S-S.S/9  U!Rx                  c  GM&  [T        Rp                  Rs                  U!Rx                  5        GMR     g)1aq  
Args:
    cfg: Model architecture configuration.
    num_classes: Number of classifier classes.
    in_chans: Number of input channels.
    global_pool: Global pooling type.
    output_stride: Output stride of network, one of (8, 16, 32).
    drop_rate: Dropout rate.
    drop_path_rate: Stochastic depth drop-path rate.
    **kwargs: Extra kwargs overlayed onto cfg.
r   Fz3Please add non-linearity constants for activation (z).)rP   )eps)rP   r   Nr   )rt   r=   T)	stagewiser   r.   rv   )r   rv   ro   rp   r%   r   r0   rq   rr   rs   r)   r1   r4   r5   r6   r:   r*   r-   r=   rt   r   zstages.r   
final_convr\   )	pool_typer   fcr   g{Gz?fan_inlinear)modenonlinearityr>   )?rS   rT   r   r   r   grad_checkpointingr   r=   _nonlin_gammar8   r   r   r7   rk   r   r9   r   r*   r   r+   r   r(   r$   r/   r4   r   r'   r   feature_infor   r#   r   ranger   r%   r)   r5   r1   r6   r:   r-   r   r{   r   stagesr2   r   r~   	final_acthead_hidden_sizer   headnamed_modules
isinstanceLinearr;   initzeros_weightnormal_biasConv2dkaiming_normal_)#rV   r   r   r   r   r   r   r   ry   rz   r   r   rt   r=   r*   r(   r   	stem_featdrop_path_ratesprev_chs
net_striderr   expected_varr   	stage_idxstage_depthrq   rs   blocks	block_idxfirst_blockrp   nmrW   s#                                     rK   rT   NormFreeNet.__init__  s   0 	/& ""'c$V$}}-v1dehererdssu/vv-,/,<,<(/
&s}}M#--<XYI 1A1ABJ%cmm4I =3OUXUeUefJMP^^WXcnn5IIae
!3<<#B3<<?cFVFV"VX[XbXbc,7MM-
 "-
 -
)	;	 'K3NJJZ^_ 
&/

&;"I{#q.[1_Q!F]*vzF"& J"*f"4Q!NF"3::i#89	'1n?a(i)@3CSCS)SUXU_U_`= #-4)) lc11 &/!^6	
 & $2  #~~ (+ww;CDTDT ::   #~~ !\\  * "mm (   *!" $39#=i#H%  ( >#%L		Q.!)"7 :8 $x:X_`i_jVk"l!mmr}}f-..FM '<N mmV, .s/?/?#BRBR/RTWT^T^ _D(43D3DaN2NDO$(1B1Bjcm$oDb! (D kkmDO"3+;+;a+?@ $ 1 1"
 "nn	

 
	 &&(DAqqyZ29955##GGNN188,GGOOAHHb#666%GGNN166* &Aryy))''xh'W66%GGNN166* )rJ   coarserZ   c                 4    [        SU(       a  SOSS4S/S9nU$ )z"Group parameters for optimization.z^stemz^stages\.(\d+)z^stages\.(\d+)\.(\d+)N)z^final_conv)i )r   r  )r   )rV   r  matchers      rK   group_matcherNormFreeNet.group_matcher
  s.     &,"2JDQ*
 rJ   enablec                     Xl         g)z)Enable or disable gradient checkpointing.N)r   )rV   r  s     rK   set_grad_checkpointing"NormFreeNet.set_grad_checkpointing  s
     #)rJ   c                 .    U R                   R                  $ )zGet the classifier head.)r   r   )rV   s    rK   get_classifierNormFreeNet.get_classifier  s     yy||rJ   c                 F    Xl         U R                  R                  X5        g)zReset the classifier head.

Args:
    num_classes: Number of classes for new classifier.
    global_pool: Global pooling type.
N)r   r   reset)rV   r   r   s      rK   reset_classifierNormFreeNet.reset_classifier   s     '		1rJ   rY   c                 $   U R                  U5      nU R                  (       a:  [        R                  R	                  5       (       d  [        U R                  U5      nOU R                  U5      nU R                  U5      nU R                  U5      nU$ )zjForward pass through feature extraction layers.

Args:
    x: Input tensor.

Returns:
    Feature tensor.
)	r   r   rb   jitis_scriptingr   r   r   r   r^   s     rK   forward_featuresNormFreeNet.forward_features*  sj     IIaL""599+A+A+C+Ct{{A.AAAOOANN1rJ   
pre_logitsc                 P    U(       a  U R                  XS9$ U R                  U5      $ )zForward pass through classifier head.

Args:
    x: Input features.
    pre_logits: Return features before final linear layer.

Returns:
    Classification logits or features.
)r!  )r   )rV   rY   r!  s      rK   forward_headNormFreeNet.forward_head<  s&     7Atyyy2RdiiPQlRrJ   c                 J    U R                  U5      nU R                  U5      nU$ )zGForward pass.

Args:
    x: Input tensor.

Returns:
    Output logits.
)r  r#  r^   s     rK   r_   NormFreeNet.forwardH  s)     !!!$a rJ   )r   r   r   r   r   r   r   r   r   r2   r   r   )  r   avg    r   r   NNrj   )T)N)r?   r@   rA   rB   rC   r    rD   rG   rF   r   rT   rb   r  ignorerH   r	   r  r  r{   Moduler  r
   r  rc   r  r#  r_   rI   rd   re   s   @rK   r   r   p  s   0  $$!#!$&B+B+ B+ 	B+
 B+ B+ B+ "B+ B+ B+H YY	D 	T#s(^ 	 	 YY)T )T ) ) YY		  2C 2hsm 2W[ 2%,, 5<< $
Sell 
S 
S 
S %,,  rJ   r#   .r$   r)   r*   r+   c                 >    U=(       d    0 n[        U USSSUUUUS9	nU$ )aJ  Create NFNet ResNet configuration.

Args:
    depths: Number of blocks in each stage.
    channels: Channel dimensions for each stage.
    group_size: Group convolution size.
    act_layer: Activation layer type.
    attn_layer: Attention layer type.
    attn_kwargs: Attention layer arguments.

Returns:
    NFNet configuration.
r   rN   r   )	r#   r$   r'   r(   r1   r)   r=   r*   r+   )r    )r#   r$   r)   r=   r*   r+   r   s          rK   
_nfres_cfgr-  V  s:    * #K

C JrJ   c                 R    SUS   -  S-  n[        SS9n[        U USSSS	US
SUS9
nU$ )zCreate NFNet RegNet configuration.

Args:
    depths: Number of blocks in each stage.
    channels: Channel dimensions for each stage.

Returns:
    NFNet configuration.
i   r     r0   rd_ratior&   r3   g      ?g      @Tse)
r#   r$   r'   r)   r/   r1   r2   r5   r*   r+   )r   r    )r#   r$   r2   r+   r   s        rK   
_nfreg_cfgr3  z  sP     (2,&#-L$K
!C JrJ   r2  r1   	feat_multc                 j    [        US   U-  5      nUb  UO[        SS9n[        U USSUUSUUUUS9n	U	$ )a  Create NFNet configuration.

Args:
    depths: Number of blocks in each stage.
    channels: Channel dimensions for each stage.
    group_size: Group convolution size.
    bottle_ratio: Bottleneck ratio.
    feat_mult: Feature multiplier for final layer.
    act_layer: Activation layer type.
    attn_layer: Attention layer type.
    attn_kwargs: Attention layer arguments.

Returns:
    NFNet configuration.
r   r0   r0  r      T)r#   r$   r'   r(   r)   r1   r6   r2   r=   r*   r+   )rD   r   r    )
r#   r$   r)   r1   r4  r=   r*   r+   r2   r   s
             rK   
_nfnet_cfgr7    sZ    2 x|i/0L!,!8+dC>PK
!!C JrJ   r:   c                 ^    [        U USSSSSSSU[        US   S-  5      US[        SS9S	9nU$ )
zCreate DeepMind NFNet configuration.

Args:
    depths: Number of blocks in each stage.
    channels: Channel dimensions for each stage.
    act_layer: Activation layer type.
    skipinit: Use skipinit initialization.

Returns:
    NFNet configuration.
r   r6  r0   Tr   r,   r2  r0  )r#   r$   r'   r(   r)   r1   r6   r7   r8   r:   r2   r=   r*   r+   )r    rD   r   )r#   r$   r=   r:   r   s        rK   _dm_nfnet_cfgr9    sR    " "+,#&C  JrJ   dm_nfnet_f0)r   rv      r   )r#   dm_nfnet_f1)rv   r      r;  dm_nfnet_f2)r   r;     	   dm_nfnet_f3)r   r3      r=  dm_nfnet_f4)   
         dm_nfnet_f5)r;  r=  $   r?  dm_nfnet_f6)r      *      nfnet_f0nfnet_f1nfnet_f2nfnet_f3nfnet_f4nfnet_f5nfnet_f6nfnet_f7)r3      0   rB  nfnet_l0g      ?rN   r   r3   )r1  
rd_divisorr<   )r#   r4  r)   r1   r+   r=   eca_nfnet_l0eca)r#   r4  r)   r1   r*   r+   r=   eca_nfnet_l1rv   eca_nfnet_l2eca_nfnet_l3nf_regnet_b0)r   r   r;  r;  nf_regnet_b1)rv   r   r   r   nf_regnet_b2)rv   r   r3   r3   )8   p      i  )r#   r$   nf_regnet_b3)rv   rD  r@  r@  )rb  r6     i  nf_regnet_b4)rv   r;     rh  )rN         ih  nf_regnet_b5)r   r   rK  rK  )P      iP  i  nf_resnet26)rv   rv   rv   rv   nf_resnet50)r   r   r;  r   nf_resnet101)r   r      r   nf_seresnet26g      ?r0  )r#   r*   r+   nf_seresnet50nf_seresnet101nf_ecaresnet26nf_ecaresnet50nf_ecaresnet101
test_nfnet)r   r   r   r   )r)  rN   `   r6  )r#   r$   r4  r)   r1   r+   r=   variant
pretrainedr   c                 P    [         U    n[        SS9n[        [        U U4UUS.UD6$ )zCreate a NormFreeNet model.

Args:
    variant: Model variant name.
    pretrained: Load pretrained weights.
    **kwargs: Additional model arguments.

Returns:
    NormFreeNet model instance.
T)flatten_sequential)	model_cfgfeature_cfg)
model_cfgsr   r   r   )rz  r{  r   r~  r  s        rK   _create_normfreenetr  %  sE     7#I$/K   rJ   urlc                 2    U SSSSS[         [        SSSS	.UE$ )
zCreate default configuration dictionary.

Args:
    url: Model weight URL.
    **kwargs: Additional configuration options.

Returns:
    Configuration dictionary.
r'  r      r  r   r   ?bicubicz
stem.conv1zhead.fcz
apache-2.0)r  r   
input_size	pool_sizecrop_pctinterpolationmeanstd
first_conv
classifierlicenser   )r  r   s     rK   _dcfgr  <  s4     =v)%.B")  rJ   zdm_nfnet_f0.dm_in1kztimm/zmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f0-604f9c3a.pth)r;  r;  )r      r  )r      r  r  squash)	hf_hub_idr  r  r  test_input_sizer  	crop_modezdm_nfnet_f1.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f1-fc540f82.pthr  r  )r   @  r  gQ?zdm_nfnet_f2.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f2-89875923.pth)r3   r3   )r   `  r  gq=
ףp?zdm_nfnet_f3.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f3-d74ab3aa.pth)rE  rE  )r     r  gGz?zdm_nfnet_f4.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f4-0ac5b10b.pth)r=  r=  )r     r  )r      r  g;On?zdm_nfnet_f5.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f5-ecb20ab1.pth)   r  )r      r  gI+?zdm_nfnet_f6.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f6-e0f12116.pth)rK  rK  )r     r  )r   @  r  gd;O?)r  r  r  r  )rG  rG  )r     r  )r   `  r  znfnet_l0.ra2_in1kzjhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nfnet_l0_ra2-45c6688d.pth)r   rj  rj  )r  r  r  r  r  test_crop_pctzeca_nfnet_l0.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l0_ra2-e3e9ac50.pthzeca_nfnet_l1.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l1_ra2-7dce93cd.pthzeca_nfnet_l2.ra3_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l2_ra3-da781a61.pth)rh  rh  )r  r  r  r  r  r   )r  r  r  r  r  znf_regnet_b1.ra2_in1kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nf_regnet_b1_256_ra2-ad85cfef.pth)r  r  r  r  r  r  )r      r  )r     r  )r@  r@  )r     r  )r  r  znf_resnet50.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nf_resnet50_ra2-9f236009.pth)r  r  r  r  r  r  r  )r0   r0   r0   gffffff?)r      r  )rD  rD  )r  r  r  r  r  r  )rw  ztest_nfnet.r160_in1kc                     [        SSU 0UD6$ )z&NFNet-F0 (DeepMind weight compatible).r{  )r:  r  r{  r   s     rK   r:  r:         NNvNNrJ   c                     [        SSU 0UD6$ )z&NFNet-F1 (DeepMind weight compatible).r{  )r<  r  r  s     rK   r<  r<    r  rJ   c                     [        SSU 0UD6$ )z&NFNet-F2 (DeepMind weight compatible).r{  )r>  r  r  s     rK   r>  r>    r  rJ   c                     [        SSU 0UD6$ )z&NFNet-F3 (DeepMind weight compatible).r{  )rA  r  r  s     rK   rA  rA    r  rJ   c                     [        SSU 0UD6$ )z&NFNet-F4 (DeepMind weight compatible).r{  )rC  r  r  s     rK   rC  rC    r  rJ   c                     [        SSU 0UD6$ )z&NFNet-F5 (DeepMind weight compatible).r{  )rH  r  r  s     rK   rH  rH    r  rJ   c                     [        SSU 0UD6$ )z&NFNet-F6 (DeepMind weight compatible).r{  )rJ  r  r  s     rK   rJ  rJ    r  rJ   c                     [        SSU 0UD6$ )z	NFNet-F0.r{  )rN  r  r  s     rK   rN  rN         KjKFKKrJ   c                     [        SSU 0UD6$ )z	NFNet-F1.r{  )rO  r  r  s     rK   rO  rO    r  rJ   c                     [        SSU 0UD6$ )z	NFNet-F2.r{  )rP  r  r  s     rK   rP  rP    r  rJ   c                     [        SSU 0UD6$ )z	NFNet-F3.r{  )rQ  r  r  s     rK   rQ  rQ    r  rJ   c                     [        SSU 0UD6$ )z	NFNet-F4.r{  )rR  r  r  s     rK   rR  rR    r  rJ   c                     [        SSU 0UD6$ )z	NFNet-F5.r{  )rS  r  r  s     rK   rS  rS     r  rJ   c                     [        SSU 0UD6$ )z	NFNet-F6.r{  )rT  r  r  s     rK   rT  rT    r  rJ   c                     [        SSU 0UD6$ )z	NFNet-F7.r{  )rU  r  r  s     rK   rU  rU    r  rJ   c                     [        SSU 0UD6$ )zNFNet-L0b w/ SiLU.

My experimental 'light' model w/ F0 repeats, 1.5x final_conv mult, 64 group_size, .25 bottleneck & SE ratio
r{  )rX  r  r  s     rK   rX  rX    s     KjKFKKrJ   c                     [        SSU 0UD6$ )zECA-NFNet-L0 w/ SiLU.

My experimental 'light' model w/ F0 repeats, 1.5x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
r{  )rZ  r  r  s     rK   rZ  rZ         O*OOOrJ   c                     [        SSU 0UD6$ )zECA-NFNet-L1 w/ SiLU.

My experimental 'light' model w/ F1 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
r{  )r\  r  r  s     rK   r\  r\  $  r  rJ   c                     [        SSU 0UD6$ )zECA-NFNet-L2 w/ SiLU.

My experimental 'light' model w/ F2 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
r{  )r]  r  r  s     rK   r]  r]  -  r  rJ   c                     [        SSU 0UD6$ )zECA-NFNet-L3 w/ SiLU.

My experimental 'light' model w/ F3 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
r{  )r^  r  r  s     rK   r^  r^  6  r  rJ   c                     [        SSU 0UD6$ )z"Normalization-Free RegNet-B0.
    r{  )r_  r  r  s     rK   r_  r_  ?       O*OOOrJ   c                     [        SSU 0UD6$ )z"Normalization-Free RegNet-B1.
    r{  )r`  r  r  s     rK   r`  r`  F  r  rJ   c                     [        SSU 0UD6$ )z"Normalization-Free RegNet-B2.
    r{  )ra  r  r  s     rK   ra  ra  M  r  rJ   c                     [        SSU 0UD6$ )z"Normalization-Free RegNet-B3.
    r{  )re  r  r  s     rK   re  re  T  r  rJ   c                     [        SSU 0UD6$ )z"Normalization-Free RegNet-B4.
    r{  )rg  r  r  s     rK   rg  rg  [  r  rJ   c                     [        SSU 0UD6$ )z"Normalization-Free RegNet-B5.
    r{  )rk  r  r  s     rK   rk  rk  b  r  rJ   c                     [        SSU 0UD6$ )z"Normalization-Free ResNet-26.
    r{  )rn  r  r  s     rK   rn  rn  i       NNvNNrJ   c                     [        SSU 0UD6$ )z"Normalization-Free ResNet-50.
    r{  )ro  r  r  s     rK   ro  ro  p  r  rJ   c                     [        SSU 0UD6$ )z#Normalization-Free ResNet-101.
    r{  )rp  r  r  s     rK   rp  rp  w  r  rJ   c                     [        SSU 0UD6$ )zNormalization-Free SE-ResNet26.r{  )rr  r  r  s     rK   rr  rr  ~       P:PPPrJ   c                     [        SSU 0UD6$ )zNormalization-Free SE-ResNet50.r{  )rs  r  r  s     rK   rs  rs    r  rJ   c                     [        SSU 0UD6$ )z Normalization-Free SE-ResNet101.r{  )rt  r  r  s     rK   rt  rt         QJQ&QQrJ   c                     [        SSU 0UD6$ )z Normalization-Free ECA-ResNet26.r{  )ru  r  r  s     rK   ru  ru    r  rJ   c                     [        SSU 0UD6$ )z Normalization-Free ECA-ResNet50.r{  )rv  r  r  s     rK   rv  rv    r  rJ   c                     [        SSU 0UD6$ )z!Normalization-Free ECA-ResNet101.r{  )rw  r  r  s     rK   rw  rw    s     RZR6RRrJ   c                     [        SSU 0UD6$ )z%Test NFNet model for experimentation.r{  )rx  r  r  s     rK   rx  rx    s     M
MfMMrJ   )r.   )r   NNTNN))r  r  i   i   Nra   NN))rW  h      r/  )r  r     r  r6  r0   r,   r   r2  N)r  r   Tr>   rj   )r   )drC   collectionsr   dataclassesr   r   	functoolsr   typingr   r   r	   r
   r   rb   torch.nnr{   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   r   _builderr   _features_fxr   _manipulater   	_registryr   r   __all__r    r+  rM   rG   rF   rk   rm   r   rD   rH   r   r   r   r   r   r-  r3  r7  r9  r  r  r  default_cfgsr:  r<  r>  rA  rC  rH  rJ  rN  rO  rP  rQ  rR  rS  rT  rU  rX  rZ  r\  r]  r^  r_  r`  ra  re  rg  rk  rn  ro  rp  rr  rs  rt  ru  rv  rw  rx  r>   rJ   rK   <module>r     sT  $ $ *  7 7   A8 8 8 * 1 ' <'
"   2Eryy E8S   *'BII *'Z ABII A AN )-(,#<:<:<: <: X&	<:
 H%<: <: 2==#tCH~-.<:@ 		""	
			&c")) cP %;$($(04!c3h!S/! SM! 	!
 SM! d38n-! !HuS#X %S/ \a : %;!04(c3h(S/( ( 	(
 ( ( ( d38n-( (Z %;	!c3h!S/! ! 	!
 !H  >\2> ]3> ]3	>
 ^4> _5> _5> _5> |,> }-> }-> ~.> />  /!>" /#>$ /%>* sr$15I+>0 srdf@1>6 btdf@7>< btdf@=>B rdf@C>N <0O>P <0Q>R <:MNS>T <:MNU>V ><OPW>X ><OPY>^ ,/_>` ,/a>b =1c>f LTt]aObcg>h LTt]aObci>j ]tQU_cQdek>n \eQUQWXo>p \eQUQWXq>r mSWSYZs>v &73STcg$15Iw>
B $ # R] .s 3 4S> ( % e&5{]M\^jrte&
 5{]M\`ltve& 5{]M\`ltve& 5{}m^bnvxe&" 5{}m^cowy#e&* 5{}m^cowy+e&2 5{}m^cowy3e&< &]M[=e&@ &]M[Ae&D &]M[Ee&H (}m]Ie&L (}m]Me&P (}m]Qe&T (}m]Ue&X (}m]Ye&^ x]Madf_e&f U{]Madfge&n U{]Madfoe&v U{}mcfhwe&~ E}mcfhe&F E&]MfqsGe&J U A]M^ikKe&R E&]MfqsSe&V E&]MfqsWe&Z E(}mhsu[e&^ E(}mhsu_e&d 5RK8ee&f E{]M\`mxzge&n Eb[9oe&r Urk:se&t Urk:ue&v e{;we&z e{;{e&| e{;}e&~ <!/-6CCe& eP OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P OD OC OK O O OD OC OK O O PT PS P[ P P Qd Qc Qk Q Q
 Qd Qc Qk Q Q
 Rt Rs R{ R R
 Rt Rs R{ R R
 Rt Rs R{ R R
 S S S S S
 N4 N3 N; N NrJ   