
    RЦi                     :   S r SSKrSSKJr  SSKJr  S(S\4S jjr " S S\R                  5      r	S(S\4S	 jjr
 " S
 S\R                  5      rS(S\4S jjr " S S\R                  5      rS(S\4S jjr " S S\R                  5      rS(S\4S jjr " S S\R                  5      rS(S\4S jjr " S S\R                  5      rS(S\4S jjr " S S\R                  5      r " S S\R,                  5      rS(S\R.                  S\S\R.                  4S jjr " S  S!\R                  5      rS(S\R.                  S\S\R.                  4S" jjr " S# S$\R                  5      rS(S\R.                  S\S\R.                  4S% jjr " S& S'\R                  5      rg))zActivations

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalinplacec                     U(       a  U R                  U R                  5       5      $ U R                  U R                  5       5      $ )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )mul_sigmoidmulxr   s     V/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/activations.pyswishr      s/     #*166!))+AquuQYY[/AA    c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )Swish   r   c                 .   > [         TU ]  5         Xl        g Nsuper__init__r   selfr   	__class__s     r   r   Swish.__init__       r   c                 ,    [        XR                  5      $ r   )r   r   r   r   s     r   forwardSwish.forward   s    Q%%r   r   F	__name__
__module____qualname____firstlineno__boolr   r   __static_attributes____classcell__r   s   @r   r   r      s      & &r   r   c                 h    U R                  [        R                  " U 5      R                  5       5      $ )zMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
NOTE: I don't have a working inplace variant
)r	   Fsoftplustanhr
   s     r   mishr/      s$     55A##%&&r   c                   >   ^  \ rS rSrSrSS\4U 4S jjjrS rSrU =r	$ )Mish$   ziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    r   c                 "   > [         TU ]  5         g r   r   r   r   s     r   r   Mish.__init__'       r   c                     [        U5      $ r   )r/   r   s     r   r   Mish.forward*   s    Awr    r!   )
r#   r$   r%   r&   __doc__r'   r   r   r(   r)   r*   s   @r   r1   r1   $   s#       r   r1   c                 P    U(       a  U R                  5       $ U R                  5       $ r   )sigmoid_r   r
   s     r   r   r   .   s    "1::<3		3r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )Sigmoid3   r   c                 .   > [         TU ]  5         Xl        g r   r   r   s     r   r   Sigmoid.__init__4   r   r   c                 d    U R                   (       a  UR                  5       $ UR                  5       $ r   )r   r<   r   r   s     r   r   Sigmoid.forward8   s    #||qzz|<<r   r    r!   r"   r*   s   @r   r>   r>   3   s      = =r   r>   c                 P    U(       a  U R                  5       $ U R                  5       $ r   )tanh_r.   r
   s     r   r.   r.   <   s    1779-QVVX-r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )TanhA   r   c                 .   > [         TU ]  5         Xl        g r   r   r   s     r   r   Tanh.__init__B   r   r   c                 d    U R                   (       a  UR                  5       $ UR                  5       $ r   )r   rE   r.   r   s     r   r   Tanh.forwardF   s     LLqwwy6affh6r   r    r!   r"   r*   s   @r   rG   rG   A   s      7 7r   rG   c                     [         R                  " U S-   5      R                  S5      nU(       a  U R                  U5      $ U R	                  U5      $ )N      @      @)r,   relu6div_r   r	   )r   r   inners      r   
hard_swishrS   J   s;    GGAFO  $E#166%=5u5r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )	HardSwishO   r   c                 .   > [         TU ]  5         Xl        g r   r   r   s     r   r   HardSwish.__init__P   r   r   c                 ,    [        XR                  5      $ r   )rS   r   r   s     r   r   HardSwish.forwardT   s    !\\**r   r    r!   r"   r*   s   @r   rU   rU   O   s      + +r   rU   c                     U(       a0  U R                  S5      R                  SS5      R                  S5      $ [        R                  " U S-   5      S-  $ )NrN   g        rO   )add_clamp_rQ   r,   rP   r
   s     r   hard_sigmoidr^   X   sB    vvbz  R(--b11wwq2v##r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )HardSigmoid_   r   c                 .   > [         TU ]  5         Xl        g r   r   r   s     r   r   HardSigmoid.__init__`   r   r   c                 ,    [        XR                  5      $ r   )r^   r   r   s     r   r   HardSigmoid.forwardd   s    A||,,r   r    r!   r"   r*   s   @r   r`   r`   _   s      - -r   r`   c                     U(       a%  U R                  SU S-   R                  SSS9-  5      $ SU -  U S-   R                  SSS9-  $ )zHard Mish
Experimental, based on notes by Mish author Diganta Misra at
  https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
g      ?   r   )minmax)r   clampr
   s     r   	hard_mishrk   h   sL    
 vvcQUMMaQM7788Qw!a%1!444r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )HardMishs   r   c                 .   > [         TU ]  5         Xl        g r   r   r   s     r   r   HardMish.__init__t   r   r   c                 ,    [        XR                  5      $ r   )rk   r   r   s     r   r   HardMish.forwardx   s    LL))r   r    r!   r"   r*   s   @r   rm   rm   s   s      * *r   rm   c            	       ~   ^  \ rS rSrSrSS\S\S\SS4U 4S jjjrS	\	R                  S\	R                  4S
 jrSrU =r$ )PReLU|   z)Applies PReLU (w/ dummy inplace arg)
    num_parametersinitr   returnNc                     > [         TU ]  XS9  g )N)rv   rw   r4   )r   rv   rw   r   r   s       r   r   PReLU.__init__   s    Br   inputc                 B    [         R                  " XR                  5      $ r   )r,   preluweightr   r{   s     r   r   PReLU.forward   s    wwukk**r   r9   )   g      ?F)r#   r$   r%   r&   r:   intfloatr'   r   torchTensorr   r(   r)   r*   s   @r   rt   rt   |   sU    Cs Ce CT C^b C C+U\\ +ell + +r   rt   r   rx   c                 .    [         R                  " U 5      $ r   r,   gelur
   s     r   r   r      s    66!9r   c                   r   ^  \ rS rSrSrS	S\4U 4S jjjrS\R                  S\R                  4S jr	Sr
U =r$ )
GELU   LApplies the Gaussian Error Linear Units function (w/ dummy inplace arg)
    r   c                 "   > [         TU ]  5         g r   r4   r   s     r   r   GELU.__init__   r6   r   r{   rx   c                 .    [         R                  " U5      $ r   r   r   s     r   r   GELU.forward   s    vve}r   r9   r!   r#   r$   r%   r&   r:   r'   r   r   r   r   r(   r)   r*   s   @r   r   r      s9      U\\ ell  r   r   c                 ,    [         R                  " U SS9$ Nr.   )approximater   r
   s     r   	gelu_tanhr      s    66!((r   c                   r   ^  \ rS rSrSrS	S\4U 4S jjjrS\R                  S\R                  4S jr	Sr
U =r$ )
GELUTanh   r   r   c                 "   > [         TU ]  5         g r   r4   r   s     r   r   GELUTanh.__init__   r6   r   r{   rx   c                 ,    [         R                  " USS9$ r   r   r   s     r   r   GELUTanh.forward   s    vve00r   r9   r!   r   r*   s   @r   r   r      s9      1U\\ 1ell 1 1r   r   c                 :    U [         R                  " SU -  5      -  $ )NgZd;?)r   r   r
   s     r   
quick_gelur      s    u}}UQY'''r   c                   r   ^  \ rS rSrSrS	S\4U 4S jjjrS\R                  S\R                  4S jr	Sr
U =r$ )
	QuickGELU   r   r   c                 "   > [         TU ]  5         g r   r4   r   s     r   r   QuickGELU.__init__   r6   r   r{   rx   c                     [        U5      $ r   )r   r   s     r   r   QuickGELU.forward   s    %  r   r9   r!   r   r*   s   @r   r   r      s9      !U\\ !ell ! !r   r   r!   )r:   r   r   torch.nnr   r,   r'   r   Moduler   r/   r1   r   r>   r.   rG   rS   rU   r^   r`   rk   rm   rt   r   r   r   r   r   r   r   r9   r   r   <module>r      s     $Bd B&BII &'T '299 4 4
=bii =.T .
7299 764 6
+		 +$T $-")) -5$ 5*ryy *+BHH +ELL 4 ELL 299 ) ) ) )1ryy 1(%,, ( (%,, (!		 !r   