
    RЦi                         S r SSKJrJr  SSKrSSKJr  SSKJr  SSK	J
r
  SSKJr  SS	KJr  SS
KJr   " S S\R"                  5      r\
 " S S\R"                  5      5       r " S S\R"                  5      rg)av  Bilinear-Attention-Transform and Non-Local Attention

Paper: `Non-Local Neural Networks With Grouped Bilinear Attentional Transforms`
    - https://openaccess.thecvf.com/content_CVPR_2020/html/Chi_Non-Local_Neural_Networks_With_Grouped_Bilinear_Attentional_Transforms_CVPR_2020_paper.html
Adapted from original code: https://github.com/BA-Transform/BAT-Image-Classification
    )OptionalTypeN)nn)
functional   )register_notrace_module)ConvNormAct)make_divisible)_assertc                   H   ^  \ rS rSrSr      SU 4S jjrS rS rSrU =r	$ )NonLocalAttn   zSpatial NL block for image classification.

This was adapted from https://github.com/BA-Transform/BAT-Image-Classification
Their NonLocal impl inspired by https://github.com/facebookresearch/video-nonlocal-net.
c                   > XgS.n	[         T
U ]  5         Uc  [        X-  US9nU(       a  US-  OSU l        [        R
                  " X4SSSS.U	D6U l        [        R
                  " X4SSSS.U	D6U l        [        R
                  " X4SSSS.U	D6U l        [        R
                  " XA4SSSS.U	D6U l	        [        R                  " U40 U	D6U l        U R                  5         g )Ndevicedtypedivisorg      g      ?r   T)kernel_sizestridebias)super__init__r
   scaler   Conv2dtpgzBatchNorm2dnormreset_parameters)selfin_channels	use_scalerd_ratiord_channels
rd_divisorr   r   _dd	__class__s             Y/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/non_local_attn.pyr   NonLocalAttn.__init__   s     /()?TK,5[D(3
;^1SW^[]^;^1SW^[]^;^1SW^[]^;^1SW^[]^NN;5"5	    c                 r   UnU R                  U5      nU R                  U5      nU R                  U5      nUR                  5       u  pgpUR	                  XgS5      R                  SSS5      nUR	                  XgS5      nUR	                  XgS5      R                  SSS5      n[        R                  " X45      U R                  -  n
[        R                  " U
SS9n
[        R                  " X5      nUR                  SSS5      R                  XgX5      nU R                  U5      nU R                  U5      U-   nU$ )Nr      r   dim)r   r   r   sizeviewpermutetorchbmmr   Fsoftmaxreshaper   r!   )r#   xshortcutr   r   r   BCHWatts              r,   forwardNonLocalAttn.forward1   s   FF1IFF1IFF1IVVX
aFF1$$Q1-FF1FF1$$Q1-iio

*ii#IIcIIaA&&qQ2FF1IIIaL8#r.   c                 J   U R                  5        GH  u  p[        U[        R                  5      (       a}  [        R                  R                  UR                  SSS9  [        [        UR                  5       5      5      S:  a,  [        R                  R                  UR                  S5        M  M  [        U[        R                  5      (       aW  [        R                  R                  UR                  S5        [        R                  R                  UR                  S5        GM  [        U[        R                  5      (       d  GM:  [        R                  R                  UR                  S5        [        R                  R                  UR                  S5        GM     g )Nfan_outrelu)modenonlinearityr   g        r   )named_modules
isinstancer   r   initkaiming_normal_weightlenlist
parameters	constant_r   r    	GroupNorm)r#   namems      r,   r"   NonLocalAttn.reset_parametersG   s   ))+GD!RYY''''HH96 ( CtALLN+,q0GG%%affc2 1Ar~~..!!!((A.!!!&&!,Ar||,,!!!((A.!!!&&!, ,r.   )r   r!   r   r   r   r   )Tg      ?N   NN)
__name__
__module____qualname____firstlineno____doc__r   rC   r"   __static_attributes____classcell__r+   s   @r,   r   r      s0      .,- -r.   r   c                      ^  \ rS rSr\R
                  \R                  SS4S\S\S\S\\R                     S\\R                     4
U 4S jjjr
S	\4S
 jrS rSrU =r$ )BilinearAttnTransformV   Nr$   
block_sizegroups	act_layer
norm_layerc                 6  > XgS.n[         T	U ]  5         [        XS4XES.UD6U l        [        R
                  " X2U-  U-  4SUS40UD6U l        [        R
                  " X2U-  U-  4SSU40UD6U l        [        XS4XES.UD6U l        X l	        X0l
        Xl        g )Nr   r   re   rf   r   )r   r   r	   conv1r   r   conv_pconv_qconv2rc   rd   r$   )
r#   r$   rc   rd   re   rf   r   r   r*   r+   s
            r,   r   BilinearAttnTransform.__init__Y   s     / aj9jgij
iiZ(?&(HlWacdVelikliiZ(?&(HlWXZdVelikl 1o	olno
$&r.   r   c                    UR                   u  p4pV[        XV:H  S5        US::  a  U$ UR                  X4-  SSS5      nU[        R                  " X"UR
                  UR                  S9-  nUR                  X4-  XUX"5      n[        R                  " [        R                  " USSS9SS9n[        R                  " [        R                  " USSS9SS9nUR                  X4XR-  XR-  5      nU$ )	N r   r0   )r   r   r2      r1      )	shaper   r5   r7   eyer   r   catsplit)r#   r<   r   r>   r?   rc   block_size1s          r,   
resize_mat BilinearAttnTransform.resize_matm   s    ()%j
)2.6HFF15"a#		!aggahh??FF15*!7IIekk!QA.A6IIekk!QA.A6FF18r.   c                    [        UR                  S   U R                  -  S:H  S5        [        UR                  S   U R                  -  S:H  S5        UR                  u  p#pEU R                  U5      n[        R
                  " X`R                  S45      n[        R
                  " USU R                  45      nU R                  U5      R                  X R                  U R                  U R                  5      R                  5       n	U R                  U5      R                  X R                  U R                  U R                  5      R                  5       n
XR                  SSS9-  n	XR                  S	SS9-  n
U	R                  X R                  SU R                  U R                  5      R                  UR                  S5      U R                  X0R                  -  U R                  U R                  5      R                  5       n	U	R                  X#U R                  U R                  5      n	U
R                  X R                  SU R                  U R                  5      R                  UR                  S5      U R                  X0R                  -  U R                  U R                  5      R                  5       n
U
R                  X#U R                  U R                  5      n
U R                  XU R                  -  5      n	U R                  XU R                  -  5      n
U	R!                  U5      nUR!                  U
5      nU R#                  U5      nU$ )
Nr0   r   ro   r   rp   T)r3   keepdimr1   )r   rr   rc   ri   r9   adaptive_max_pool2drj   r5   rd   sigmoidrk   sumexpandr4   
contiguousrw   matmulrl   )r#   r<   r>   r?   r@   rA   outrpcpr   qys               r,   rC   BilinearAttnTransform.forwardz   sp   doo-2B7doo-2B7WW
ajjm""3!(<=""3DOO(<=KKO  KK$//RZZ\KKO  KK$//RZZ\!T**!T**FF1kk1dootGNNqvvPQ++-tQQ[Q[Q] 	
FF1$//:FF1kk1dootGNNqvvPQ++-tQQ[Q[Q] 	
FF1$//:OOADOO34OOADOO34HHQKHHQKJJqMr.   )rc   ri   rl   rj   rk   rd   r$   )rX   rY   rZ   r[   r   ReLUr    intr   Moduler   rw   rC   r]   r^   r_   s   @r,   ra   ra   V   sy     *,*,..'' ' 	'
 BII' RYY' '(s  r.   ra   c                      ^  \ rS rSrSrSSSSSS\R                  \R                  SS4
S	\S
\S\S\	S\
\   S\S\	S\\R                     S\\R                     4U 4S jjjrS rSrU =r$ )BatNonLocalAttn   zKBAT
Adapted from: https://github.com/BA-Transform/BAT-Image-Classification
   r1   g      ?NrW   g?r$   rc   rd   r&   r'   r(   	drop_ratere   rf   c                    > XS.n[         TU ]  5         Uc  [        X-  US9n[        XS4XS.UD6U l        [        UUU4UU	S.UD6U l        [        XQS4XS.UD6U l        [        R                  " US9U l
        g )Nr   r   r   rh   )r   )r   r   r
   r	   ri   ra   barl   r   	Dropout2ddropout)r#   r$   rc   rd   r&   r'   r(   r   re   rf   r   r   r)   r*   r+   s                 r,   r   BatNonLocalAttn.__init__   s     /()?TK 1o	olno
'
  !
 
 !1o	olno
||i0r.   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nX1-   $ )N)ri   r   rl   r   )r#   r<   xlr   s       r,   rC   BatNonLocalAttn.forward   s=    ZZ]GGBKJJqMLLOur.   )r   ri   rl   r   )rX   rY   rZ   r[   r\   r   r   r    r   floatr   r   r   r   rC   r]   r^   r_   s   @r,   r   r      s      ")-")+*,..11 1 	1
 1 "#1 1 1 BII1 RYY1 1> r.   r   )r\   typingr   r   r7   r   torch.nnr   r9   _fxr   conv_bn_actr	   helpersr
   trace_utilsr   r   r   ra   r    r.   r,   <module>r      sg    "   $ ( $ #  @-299 @-F :BII : :z)bii )r.   