
    RЦi3                        S r SSKrSSKJr  SSKJr  S rS r " S S\R                  R                  5      r
S)S	 jr " S
 S\R                  5      rS rS r " S S\R                  R                  5      rS)S jr " S S\R                  5      rS)S\4S jjrS r " S S\R                  R                  5      rS)S\4S jjr " S S\R                  5      rS rS r " S S\R                  R                  5      rS)S jr " S  S!\R                  5      rS" rS# r " S$ S%\R                  R                  5      r S)S\4S& jjr! " S' S(\R                  5      r"g)*a}  Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use
basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalc                 L    U R                  [        R                  " U 5      5      $ N)multorchsigmoidxs    Y/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/activations_me.py	swish_fwdr      s    55q!""    c                 N    [         R                  " U 5      nXSU SU-
  -  -   -  -  $ N   )r   r	   )r   grad_output	x_sigmoids      r   	swish_bwdr      s,    a Iq1I+>'>?@@r   c                   H    \ rS rSrSr\S 5       r\S 5       r\S 5       rSr	g)SwishAutoFn   zoptimised Swish w/ memory-efficient checkpoint
Inspired by conversation btw Jeremy Howard & Adam Pazske
https://twitter.com/jeremyphoward/status/1188251041835315200
c                 F    U R                  SXR                  SU5      5      $ )NMulSigmoid)op)gr   s     r   symbolicSwishAutoFn.symbolic   s    ttE1dd9a011r   c                 :    U R                  U5        [        U5      $ r   )save_for_backwardr   ctxr   s     r   forwardSwishAutoFn.forward#   s    a |r   c                 6    U R                   S   n[        X!5      $ Nr   )saved_tensorsr   r"   r   r   s      r   backwardSwishAutoFn.backward(   s    a ((r    N)
__name__
__module____qualname____firstlineno____doc__staticmethodr   r#   r)   __static_attributes__r+   r   r   r   r      sC     2 2   ) )r   r   c                 ,    [         R                  U 5      $ r   r   applyr   inplaces     r   swish_mer8   .   s    Qr   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )SwishMe2   r7   c                 "   > [         TU ]  5         g r   super__init__selfr7   	__class__s     r   r?   SwishMe.__init__3       r   c                 ,    [         R                  U5      $ r   r4   rA   r   s     r   r#   SwishMe.forward6   s      ##r   r+   F	r,   r-   r.   r/   boolr?   r#   r2   __classcell__rB   s   @r   r:   r:   2   s      $ $r   r:   c                 t    U R                  [        R                  " [        R                  " U 5      5      5      $ r   )r   r   tanhFsoftplusr
   s    r   mish_fwdrQ   :   s"    55AJJqM*++r   c                     [         R                  " U 5      n[        R                  " U 5      R	                  5       nUR                  X0U-  SX3-  -
  -  -   5      $ r   )r   r	   rO   rP   rN   r   )r   r   r   	x_tanh_sps       r   mish_bwdrT   >   sI    a I

1""$I??99}I<Q8Q'RRSSr   c                   8    \ rS rSrSr\S 5       r\S 5       rSrg)
MishAutoFnD   zMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
A memory efficient variant of Mish
c                 :    U R                  U5        [        U5      $ r   )r    rQ   r!   s     r   r#   MishAutoFn.forwardH   s    a {r   c                 6    U R                   S   n[        X!5      $ r&   )r'   rT   r(   s      r   r)   MishAutoFn.backwardM   s    a ''r   r+   N	r,   r-   r.   r/   r0   r1   r#   r)   r2   r+   r   r   rV   rV   D   s/       ( (r   rV   c                 ,    [         R                  U 5      $ r   rV   r5   r6   s     r   mish_mer_   S   s    Ar   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )MishMeW   r7   c                 "   > [         TU ]  5         g r   r=   r@   s     r   r?   MishMe.__init__X   rD   r   c                 ,    [         R                  U5      $ r   r^   rF   s     r   r#   MishMe.forward[   s    ""r   r+   rH   rI   rL   s   @r   ra   ra   W   s      # #r   ra   r7   c                 F    U S-   R                  SSS9R                  S5      $ N   r      minmax      @clampdivr6   s     r   hard_sigmoid_fwdrr   _   s$    E==QA=&**2..r   c                 T    [         R                  " U 5      U S:  U S:*  -  -  S-  nX-  $ )N            @rn   )r   	ones_liker   r   ms      r   hard_sigmoid_bwdry   c   s/    qCxAG45:A?r   c                   4    \ rS rSr\S 5       r\S 5       rSrg)HardSigmoidAutoFnh   c                 :    U R                  U5        [        U5      $ r   )r    rr   r!   s     r   r#   HardSigmoidAutoFn.forwardi   s    a ""r   c                 6    U R                   S   n[        X!5      $ r&   )r'   ry   r(   s      r   r)   HardSigmoidAutoFn.backwardn   s    a //r   r+   N)r,   r-   r.   r/   r1   r#   r)   r2   r+   r   r   r{   r{   h   s(    # # 0 0r   r{   c                 ,    [         R                  U 5      $ r   r{   r5   r6   s     r   hard_sigmoid_mer   t   s    ""1%%r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )HardSigmoidMex   r7   c                 "   > [         TU ]  5         g r   r=   r@   s     r   r?   HardSigmoidMe.__init__y   rD   r   c                 ,    [         R                  U5      $ r   r   rF   s     r   r#   HardSigmoidMe.forward|   s     &&q))r   r+   rH   rI   rL   s   @r   r   r   x   s      * *r   r   c                 J    X S-   R                  SSS9R                  S5      -  $ rh   ro   r
   s    r   hard_swish_fwdr      s'    A}}}*..r222r   c                     [         R                  " U 5      U S:  -  n[         R                  " U S:  U S:*  -  U S-  S-   U5      nX-  $ )Nru   rt         ?r   rv   whererw   s      r   hard_swish_bwdr      sH    a2g&AQ#X!r'*QVb[!<A?r   c                   H    \ rS rSrSr\S 5       r\S 5       r\S 5       rSr	g)HardSwishAutoFn   z'A memory efficient HardSwish activationc                 :    U R                  U5        [        U5      $ r   )r    r   r!   s     r   r#   HardSwishAutoFn.forward   s    a a  r   c                 6    U R                   S   n[        X!5      $ r&   )r'   r   r(   s      r   r)   HardSwishAutoFn.backward   s    a a--r   c                    U R                  SXR                  S[        R                  " S[        R                  S9S95      nU R                  SX R                  S[        R                  " S[        R                  S9S9U R                  S[        R                  " S[        R                  S9S95      nU R                  S	X0R                  S[        R                  " S[        R                  S9S95      nU R                  S
X5      $ )NAddConstantri   )dtype)value_tClipr   rj   Divr   )r   r   tensorfloat)r   rA   input	hardtanh_s       r   r   HardSwishAutoFn.symbolic   s    UD$$z5<<QVQ\Q\;]$"^_DDZaW\WbWbAc(dfgfjfjku  @E  @L  @L  MN  V[  Va  Va  @bfj  gc  d	DD	44
ELLQRZ_ZeZeDf4+gh	ttE4++r   r+   N)
r,   r-   r.   r/   r0   r1   r#   r)   r   r2   r+   r   r   r   r      s?    1! ! . . , ,r   r   c                 ,    [         R                  U 5      $ r   r   r5   r6   s     r   hard_swish_mer      s      ##r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )HardSwishMe   r7   c                 "   > [         TU ]  5         g r   r=   r@   s     r   r?   HardSwishMe.__init__   rD   r   c                 ,    [         R                  U5      $ r   r   rF   s     r   r#   HardSwishMe.forward   s    $$Q''r   r+   rH   rI   rL   s   @r   r   r      s      ( (r   r   c                 4    SU -  U S-   R                  SSS9-  $ )Nr      r   rk   )rp   r
   s    r   hard_mish_fwdr      s"    7a!e]]qa]000r   c                     [         R                  " U 5      U S:  -  n[         R                  " U S:  U S:*  -  U S-   U5      nX-  $ )Ng       g        g      ?r   rw   s      r   hard_mish_bwdr      sD    a3h'AQ#X!r'*AFA6A?r   c                   8    \ rS rSrSr\S 5       r\S 5       rSrg)HardMishAutoFn   zA memory efficient variant of Hard Mish
Experimental, based on notes by Mish author Diganta Misra at
  https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
c                 :    U R                  U5        [        U5      $ r   )r    r   r!   s     r   r#   HardMishAutoFn.forward   s    a Qr   c                 6    U R                   S   n[        X!5      $ r&   )r'   r   r(   s      r   r)   HardMishAutoFn.backward   s    a Q,,r   r+   Nr\   r+   r   r   r   r      s/         - -r   r   c                 ,    [         R                  U 5      $ r   r   r5   r6   s     r   hard_mish_mer      s    ""r   c                   :   ^  \ rS rSrSS\4U 4S jjjrS rSrU =r$ )
HardMishMe   r7   c                 "   > [         TU ]  5         g r   r=   r@   s     r   r?   HardMishMe.__init__   rD   r   c                 ,    [         R                  U5      $ r   r   rF   s     r   r#   HardMishMe.forward   s    ##A&&r   r+   rH   rI   rL   s   @r   r   r      s      ' 'r   r   rH   )#r0   r   r   torch.nnr   rO   r   r   autogradFunctionr   r8   Moduler:   rQ   rT   rV   r_   ra   rJ   rr   ry   r{   r   r   r   r   r   r   r   r   r   r   r   r   r+   r   r   <module>r      s5  	   $#A
)%..)) )( $bii $,T((( (#RYY #/ /
	0// 	0& &*BII *3,enn-- ,($(")) (1-U^^,, - #T #' 'r   