
    RЦi)                     p   S r SSKJr  SSKJrJrJrJr  SSKJ	r	  SSK
Jr  SSKJr   " S S	\	R                  5      r " S
 S\	R                  5      r\" \\	R"                  SS9r " S S\	R                  5      r " S S\	R                  5      r " S S\	R                  5      r " S S\	R                  5      rg)zkMLP module w/ dropout and configurable activation layer

Hacked together by / Copyright 2020 Ross Wightman
    )partial)OptionalTypeUnionTuple)nn   )GlobalResponseNorm)	to_2tuplec                      ^  \ rS rSrSrSS\R                  SSSSSS4	S\S\\   S	\\   S
\	\R                     S\\	\R                        S\\\\\4   4   S\\\\\4   4   S\4U 4S jjjrS rSrU =r$ )Mlp   zMLP as used in Vision Transformer, MLP-Mixer and related networks

NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected.
NT        Fin_featureshidden_featuresout_features	act_layer
norm_layerbiasdropuse_convc                   > XS.n[         TU ]  5         U=(       d    UnU=(       d    Un[        U5      n[        U5      nU(       a  [        [        R
                  SS9O[        R                  nU" X4SUS   0UD6U l        U" 5       U l        [        R                  " US   5      U l
        Ub	  U" U40 UD6O[        R                  " 5       U l        U" X#4SUS   0UD6U l        [        R                  " US   5      U l        g )Ndevicedtyper	   kernel_sizer   r   )super__init__r   r   r   Conv2dLinearfc1actDropoutdrop1Identitynormfc2drop2)selfr   r   r   r   r   r   r   r   r   r   dd
drop_probslinear_layer	__class__s                 N/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/mlp.pyr   Mlp.__init__   s     /#2{)8[t_
<Dwryya8"))Q47QbQ;ZZ
1.
9C9OJ5"5UWU`U`Ub	RDGRrRZZ
1.
    c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU$ N)r"   r#   r%   r'   r(   r)   r*   xs     r/   forwardMlp.forward/   sU    HHQKHHQKJJqMIIaLHHQKJJqMr1   )r#   r%   r)   r"   r(   r'   __name__
__module____qualname____firstlineno____doc__r   GELUintr   r   Moduler   boolr   floatr   r6   __static_attributes____classcell__r.   s   @r/   r   r      s     .2*.)+483768"// &c]/ #3-	/
 BII/ !bii1/ eD$J//0/ uUE\223/ / /8 r1   r   c                      ^  \ rS rSrSrSS\R                  SSSSSSS4
S\S\\   S	\\   S
\	\R                     S\\	\R                        S\\\\\4   4   S\\\\\4   4   S\S\4U 4S jjjrS rS rSrU =r$ )GluMlp9   zMLP w/ GLU style gating
See: https://arxiv.org/abs/1612.08083, https://arxiv.org/abs/2002.05202

NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected.
NTr   Fr   r   r   r   r   r   r   r   	gate_lastc                 X  > XS.n[         TU ]  5         U=(       d    UnU=(       d    UnUS-  S:X  d   e[        U5      n[        U5      nU(       a  [        [        R
                  SS9O[        R                  nU(       a  SOSU l        Xl        U" X4SUS   0UD6U l	        U" 5       U l
        [        R                  " US   5      U l        Ub  U" US-  40 UD6O[        R                  " 5       U l        U" US-  U4SUS   0UD6U l        [        R                  " US   5      U l        g )Nr      r   r	   r   r   )r   r   r   r   r   r    r!   	chunk_dimrI   r"   r#   r$   r%   r&   r'   r(   r)   )r*   r   r   r   r   r   r   r   r   rI   r   r   r+   r,   r-   r.   s                  r/   r   GluMlp.__init__?   s    /#2{)8["a'''t_
<Dwryya8"))&B"Q47QbQ;ZZ
1.
>H>TJ!3:r:Z\ZeZeZg	1 4lWaWTVWZZ
1.
r1   c                    U R                   R                  bZ  [        R                  R	                  U R                   R                  U R                   R                  R
                  S   S-  S  5        [        R                  R                  U R                   R                  U R                   R                  R
                  S   S-  S  SS9  g )Nr   rK   ư>std)r"   r   r   initones_shapenormal_weightr*   s    r/   init_weightsGluMlp.init_weights_   s    88==$GGMM$((--(;(;A(>!(C(DEF
(=(=a(@A(E(FGTRr1   c                 X   U R                  U5      nUR                  SU R                  S9u  p#U R                  (       a  X R	                  U5      -  OU R	                  U5      U-  nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ )NrK   )dim)	r"   chunkrM   rI   r#   r%   r'   r(   r)   )r*   r5   x1x2s       r/   r6   GluMlp.forwarde   s    HHQK/!%B"TXXb\B5FJJqMIIaLHHQKJJqMr1   )r#   rM   r%   r)   r"   r(   rI   r'   )r9   r:   r;   r<   r=   r   Sigmoidr?   r   r   r@   r   rA   r   rB   r   rY   r6   rC   rD   rE   s   @r/   rG   rG   9   s     .2*.)+483768""// &c]/ #3-	/
 BII/ !bii1/ eD$J//0/ uUE\223/ / / /@S r1   rG   F)r   rI   c                      ^  \ rS rSrSrSS\R                  SSSSSS4	S\S\\   S	\\   S
\	\R                     S\\	\R                        S\\\\\4   4   S\\\\\4   4   S\4U 4S jjjrS rS rSrU =r$ )SwiGLUs   zSwiGLU
NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
better matches some other common impl which makes mapping checkpoints simpler.
NTr   r   r   r   r   r   r   r   r   align_toc                 B  > XS.n[         TU ]  5         U=(       d    UnU=(       d    Un[        U5      n[        U5      nU(       a  X"* U-  -   n[        R                  " X4SUS   0UD6U l        [        R                  " X4SUS   0UD6U l        U" 5       U l        [        R                  " US   5      U l	        Ub	  U" U40 UD6O[        R                  " 5       U l        [        R                  " X#4SUS   0UD6U l        [        R                  " US   5      U l        g )Nr   r   r   r	   )r   r   r   r   r!   fc1_gfc1_xr#   r$   r%   r&   r'   r(   r)   )r*   r   r   r   r   r   r   r   re   r   r   r+   r,   r.   s                r/   r   SwiGLU.__init__x   s     /#2{)8[t_
-1AH1LMOYY{P$q'PRP
YY{P$q'PRP
;ZZ
1.
9C9OJ5"5UWU`U`Ub	99_OaOBOZZ
1.
r1   c                     U R                   R                  b3  [        R                  R	                  U R                   R                  5        [        R                  R                  U R                   R                  SS9  g )NrP   rQ   )rg   r   r   rS   rT   rV   rW   rX   s    r/   rY   SwiGLU.init_weights   sF    ::??&GGMM$**//*


))t4r1   c                     U R                  U5      nU R                  U5      nU R                  U5      U-  nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  U5      nU$ r3   )rg   rh   r#   r%   r'   r(   r)   )r*   r5   x_gates      r/   r6   SwiGLU.forward   sh    AJJqMHHVq JJqMIIaLHHQKJJqMr1   )r#   r%   r)   rg   rh   r(   r'   )r9   r:   r;   r<   r=   r   SiLUr?   r   r   r@   r   rA   r   rB   r   rY   r6   rC   rD   rE   s   @r/   rc   rc   s   s     .2*.)+483768// &c]/ #3-	/
 BII/ !bii1/ eD$J//0/ uUE\223/ / />5 r1   rc   c                     ^  \ rS rSrSrSS\R                  SSSSSS4	S\S\\   S\\   S	\	\R                     S
\\	\R                        S\\	\R                        S\\\\\4   4   S\\\\\4   4   4U 4S jjjrS rSrU =r$ )GatedMlp   zMLP as used in gMLP
    NTr   r   r   r   r   r   
gate_layerr   r   c                 Z  > XS.n[         TU ]  5         U=(       d    UnU=(       d    Un[        U5      n[        U5      n[        R                  " X4SUS   0UD6U l        U" 5       U l        [        R                  " US   5      U l        Ub  US-  S:X  d   eU" U40 UD6U l	        US-  nO[        R                  " 5       U l	        Ub	  U" U40 UD6O[        R                  " 5       U l        [        R                  " X#4SUS   0UD6U l        [        R                  " US   5      U l        g )Nr   r   r   rK   r	   )r   r   r   r   r!   r"   r#   r$   r%   gater&   r'   r(   r)   )r*   r   r   r   r   r   rs   r   r   r   r   r+   r,   r.   s                r/   r   GatedMlp.__init__   s    /#2{)8[t_
99[NQN2N;ZZ
1.
!"Q&!+++"?9b9DI-2ODI9C9OJ5"5UWU`U`Ub	99_OaOBOZZ
1.
r1   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  U5      nU$ r3   )r"   r#   r%   ru   r'   r(   r)   r4   s     r/   r6   GatedMlp.forward   sb    HHQKHHQKJJqMIIaLIIaLHHQKJJqMr1   )r#   r%   r)   r"   r(   ru   r'   r8   rE   s   @r/   rq   rq      s    
 .2*.)+48483768// &c]/ #3-	/
 BII/ !bii1/ !bii1/ eD$J//0/ uUE\223/ /B r1   rq   c                      ^  \ rS rSrSrSS\R                  SSSSS4S\S\\   S\\   S	\	\R                     S
\\	\R                        S\\\\\4   4   S\4U 4S jjjrS rSrU =r$ )ConvMlp   zFMLP using 1x1 convs that keeps spatial dims (for 2D NCHW tensors)
    NTr   r   r   r   r   r   r   r   c
                   > XS.n
[         TU ]  5         U=(       d    UnU=(       d    Un[        U5      n[        R                  " X4SUS   S.U
D6U l        U(       a	  U" U40 U
D6O[        R                  " 5       U l        U" 5       U l        [        R                  " U5      U l
        [        R                  " X#4SUS   S.U
D6U l        g )Nr   r	   r   )r   r   )r   r   r   r   r    r"   r&   r'   r#   r$   r   r(   )r*   r   r   r   r   r   r   r   r   r   r+   r.   s              r/   r   ConvMlp.__init__   s     /#2{)8[99[]qtTUw]Z\]9CJ5"5	;JJt$	99_^PTUVPW^[]^r1   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU$ r3   )r"   r'   r#   r   r(   r4   s     r/   r6   ConvMlp.forward   sH    HHQKIIaLHHQKIIaLHHQKr1   )r#   r   r"   r(   r'   )r9   r:   r;   r<   r=   r   ReLUr?   r   r   r@   r   rA   r   rB   r   r6   rC   rD   rE   s   @r/   rz   rz      s    
 .2*.)+4837__ &c]_ #3-	_
 BII_ !bii1_ eD$J//0_ _ _0 r1   rz   c                      ^  \ rS rSrSrSS\R                  SSSSS4S\S\\   S	\\   S
\	\R                     S\\\\\4   4   S\\\\\4   4   S\4U 4S jjjrS rSrU =r$ )GlobalResponseNormMlp   zMLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d

NOTE: Intended for '2D' NCHW (use_conv=True) or NHWC (use_conv=False, channels-last) tensor layouts
NTr   Fr   r   r   r   r   r   r   c
                   > XS.n
[         TU ]  5         U=(       d    UnU=(       d    Un[        U5      n[        U5      nU(       a  [        [        R
                  SS9O[        R                  nU" X4SUS   0U
D6U l        U" 5       U l        [        R                  " US   5      U l
        [        U4SU(       + 0U
D6U l        U" X#4SUS   0U
D6U l        [        R                  " US   5      U l        g )Nr   r	   r   r   r   channels_last)r   r   r   r   r   r    r!   r"   r#   r$   r%   r
   grnr(   r)   )r*   r   r   r   r   r   r   r   r   r   r+   r,   r-   r.   s                r/   r   GlobalResponseNormMlp.__init__   s     /#2{)8[t_
<Dwryya8"))Q47QbQ;ZZ
1.
%oX\XUWXRDGRrRZZ
1.
r1   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU$ r3   )r"   r#   r%   r   r(   r)   r4   s     r/   r6   GlobalResponseNormMlp.forward  sU    HHQKHHQKJJqMHHQKHHQKJJqMr1   )r#   r%   r)   r"   r(   r   r8   rE   s   @r/   r   r      s     .2*.)+3768"// &c]/ #3-	/
 BII/ eD$J//0/ uUE\223/ / /6 r1   r   N)r=   	functoolsr   typingr   r   r   r   torchr   r   r
   helpersr   r@   r   rG   ro   SwiGLUPackedrc   rq   rz   r    r1   r/   <module>r      s     / /  # (")) (V4RYY 4n vEB2RYY 2j,ryy ,^!bii !H'BII 'r1   