
    RЦi/                         S r SSKJrJrJr  SSKrSSKJr  SSKJ	r	  SSK
Jr  SSKJr  SSKJrJr  SS	KJr   " S
 S\R&                  5      r " S S\R&                  5      rg)a9  Attention Pool 2D

Implementations of 2D spatial feature pooling using multi-head attention instead of average pool.

Based on idea in CLIP by OpenAI, licensed Apache 2.0
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

Hacked together by / Copyright 2021 Ross Wightman
    )OptionalUnionTupleN   )use_fused_attn)	to_2tuple)resample_abs_pos_embed)apply_rot_embed_catcreate_rope_embed)trunc_normal_c                   l  ^  \ rS rSr% Sr\R                  R                  \   \	S'                SS\
S\\
   S\\
\\
\
4   4   S\\
   S\\
   S	\\
   S
\S\S\S\S\S\4U 4S jjjrSS\4S jjrSS\\
   S\\   4S jjrS\R&                  S\
S\
S\R&                  4S jrSS\4S jjrSrU =r$ ) RotAttentionPool2d   a)  Attention based 2D feature pooling w/ rotary (relative) pos embedding.
This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

Adapted from the AttentionPool2d in CLIP w/ rotary embedding instead of learned embed.
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

NOTE: While this impl does not require a fixed feature size, performance at differeing resolutions from
train varies widely and falls off dramatically. I'm not sure if there is a way around this... -RW

fused_attnin_featuresout_featuresref_feat_size	embed_dimhead_dim	num_headsqkv_biasqkv_separate	pool_typeclass_token	drop_rate	rope_typec           
        > XS.n[         TU ]  5         U	S;   d   eU=(       d    U=U l        nXl        U=(       d    UU l        [        U5      nUb  XF-  S:X  d   eXF-  nOXE-  S:X  d   eXE-  nX`l        XPl        U	R                  5       U l	        U R                  S-  U l
        [        5       U l        Xl        U
(       a2  [        R                  " [         R"                  " SU40 UD65      U l        OS U l        U(       ab  [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        S U l        O![        R&                  " XS-  4SU0UD6U l        [        R0                  " U5      U l        [        R&                  " X@R                  40 UD6U l        [7        S
UUUSUSS	.UD6U l        g )Ndevicedtype tokenr         r   bias   F)r   dimr   	in_pixelsref_feat_shaperotate_half )super__init__r   r   r   r   r   r   lowerr   scaler   r   r   nn	Parametertorchzeros	cls_tokenLinearqkvqkvDropoutdropprojr   	pos_embed)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r    dd	__class__s                   [/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/attention_pool2d.pyr-   RotAttentionPool2d.__init__"   s   " /M)))%.%=+=&(7K!-0 (A--- -H'1,,,!-I" "*]]d*
(*"\\%++a*Ib*IJDN!DNYY{KHKKDFYY{KHKKDFYY{KHKKDFDHyy!mQ(QbQDHJJy)	IIi):):AbA	* 
(
 
    zero_init_lastc                    U R                   Gc  U R                  R                  n[        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        g U R                   R                  n[        U R                   R                  US-  S9  [
        R                  R                  U R                   R                  5        g Nr$   )std)r9   r6   r   r   weightr0   initzeros_r%   r7   r8   r>   rD   r   s      rA   init_weightsRotAttentionPool2d.init_weights`   s    88&&,,K$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'((..K$((//{d/BCGGNN488==)rC   num_classesc                     Ub  US;   d   eX l         Ub[  US:  a!  [        R                  " U R                  U5      O[        R                  " 5       U l        US:  a  UOU R                  U l        g g Nr!   r   r   r0   r5   r   Identityr<   r   r   r>   rN   r   s      rA   resetRotAttentionPool2d.resetn   e     ---&N"DORSO		$"2"2K@Y[YdYdYfDI/:QDNND #rC   xHWreturnc                     U R                   S:X  a  US S 2S4   nU$ US S 2SS 24   R                  UR                  S   X#S5      R                  SSSS5      nU$ Nr#   r   r   r&      r   reshapeshapepermuter>   rW   rX   rY   s       rA   _poolRotAttentionPool2d._poolw   `    >>W$!Q$A  !QR%  Q26>>q!QJArC   
pre_logitsc                 `   UR                   u  p4pVXV-  nUR                  S5      R                  SS5      nU R                  c&  [        R
                  " UR                  SSS9U/SS9nO?[        R
                  " U R                  R                  UR                   S   SS5      U/SS9nU R                  c  U R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      nU R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n	U R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n
OaU R                  U5      R                  X7S-   SU R                  U R                  5      R                  SSSSS	5      nUR!                  S5      u  pn
U R"                  R%                  XV45      n['        U[(        5      (       a  [        R
                  " USS9n[        R
                  " US S 2S S 2S S2S S 24   [+        US S 2S S 2SS 2S S 24   U5      /SS9R-                  U
5      n[        R
                  " U	S S 2S S 2S S2S S 24   [+        U	S S 2S S 2SS 2S S 24   U5      /SS9R-                  U
5      n	U R.                  (       a!  [0        R2                  R5                  XU
5      nO5XR6                  -  nXR                  S
S5      -  nUR9                  SS9nX-  nUR                  SS5      R                  X7S-   S5      nU R;                  U5      nU(       a  U R=                  XU5      nU$ U R?                  U5      nU R=                  XU5      nU$ )Nr^   r   Tkeepdimr'   r   r]   r&      ) ra   flatten	transposer4   r2   catmeanexpandr9   r6   r`   r   r   r7   r8   rb   unbindr=   	get_embed
isinstancetupler
   type_asr   r0   
functionalscaled_dot_product_attentionr/   softmaxr;   rd   r<   )r>   rW   rg   B_rX   rY   Nr6   r7   r8   ropeattns                rA   forwardRotAttentionPool2d.forward   s   WW
aEIIaL""1a(>>!		166!T62A6A>A		4>>00RDaHaPA88q	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZA##A1uaOWWXY[\^_abdefAhhqkGA!~~''/dE""99Tr*DIIqArr1~':1Q12q[>4'PQWXYaabcdIIqArr1~':1Q12q[>4'PQWXYaabcd??::1CAJJA{{2r**D<<B<'DAKK1%%aQ3IIaL

1#AHIIaLJJqQrC   )r4   r;   r   r   r   r   r7   r   r   r   r=   r<   r6   r9   r   r/   r8   )N   N@   NTFr#   F        rp   NNFNN)__name__
__module____qualname____firstlineno____doc__r2   jitFinalbool__annotations__intr   r   r   strfloatr-   rL   rT   Tensorrd   r   __static_attributes____classcell__r@   s   @rA   r   r      sM    		%%
 +/9:'+&('+!!&$ %!"<
<
 #3-<
 !eCHo!56	<

  }<
 sm<
  }<
 <
 <
 <
 <
 <
 <
 <
|*4 *S# S(3- Su||    %T % %rC   r   c                   f  ^  \ rS rSr% Sr\R                  R                  \   \	S'               SS\
S\\
\\
\
4   4   S\\
   S\\
   S\\
   S	\\
   S
\S\S\S\S\4U 4S jjjrSS\4S jjrSS\\
   S\\   4S jjrS\R&                  S\
S\
S\R&                  4S jrSS\4S jjrSrU =r$ )AttentionPool2d   a  Attention based 2D feature pooling w/ learned (absolute) pos embedding.
This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

It was based on impl in CLIP by OpenAI
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

NOTE: This requires feature size upon construction and well prevent adaptive sizing of the network.
r   r   	feat_sizer   r   r   r   r   r   r   r   r   c                 x  > XS.n[         TU ]  5         U	S;   d   eU=(       d    U=U l        nXl        U=(       d    UU l        Ub  XF-  S:X  d   eXF-  nOXE-  S:X  d   eXE-  n[        U5      U l        U R                  S   U R                  S   -  U l        X`l        XPl	        Xl
        U R                  S-  U l        [        5       U l        U
(       a2  [        R                  " [         R"                  " SU40 UD65      U l        OS U l        U(       ab  [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        S U l        O6S =U l        =U l        U l        [        R&                  " XS-  4SU0UD6U l        [        R0                  " U5      U l        [        R&                  " X@R                  40 UD6U l        [        R                  " [         R"                  " U R                  S-   U40 UD65      U l        U R9                  5         g )Nr   r!   r   r   r$   r%   r&   )r,   r-   r   r   r   r   r   seq_lenr   r   r   r/   r   r   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   rL   )r>   r   r   r   r   r   r   r   r   r   r   r   r   r    r?   r@   s                  rA   r-   AttentionPool2d.__init__   s     /M)))%.%=+=&(7K (A--- -H'1,,,!-I"9-~~a(4>>!+<<" "]]d*
(*\\%++a*Ib*IJDN!DNYY{KHKKDFYY{KHKKDFYY{KHKKDFDH'++DF+TVdfyy!mQ(QbQDHJJy)	IIi):):AbA	ekk$,,2BK&VSU&VWrC   rD   c                 H   U R                   Gc  U R                  R                  n[        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        OjU R                   R                  n[        U R                   R                  US-  S9  [
        R                  R                  U R                   R                  5        [        U R                  US-  S9  g rF   )r9   r6   r   r   rH   r0   rI   rJ   r%   r7   r8   r=   rK   s      rA   rL   AttentionPool2d.init_weights   s    88&&,,K$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'((..K$((//{d/BCGGNN488==)dnn+*=>rC   rN   c                     Ub  US;   d   eX l         Ub[  US:  a!  [        R                  " U R                  U5      O[        R                  " 5       U l        US:  a  UOU R                  U l        g g rP   rQ   rS   s      rA   rT   AttentionPool2d.reset   rV   rC   rW   rX   rY   rZ   c                     U R                   S:X  a  US S 2S4   nU$ US S 2SS 24   R                  UR                  S   X#S5      R                  SSSS5      nU$ r\   r_   rc   s       rA   rd   AttentionPool2d._pool  rf   rC   rg   c                    UR                   u  p4pVXV-  nUR                  S5      R                  SS5      nU R                  c&  [        R
                  " UR                  SSS9U/SS9nO?[        R
                  " U R                  R                  UR                   S   SS5      U/SS9n[        U R                  R                  S5      XV4SS9nX-   nU R                  c  U R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n	U R!                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n
U R#                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      nO_U R                  U5      R                  USS	U R                  U R                  5      R%                  SSS	SS
5      nUR'                  S5      u  pnU R(                  (       a!  [*        R,                  R/                  XU5      nO5XR0                  -  n	XR                  SS5      -  nUR3                  SS9nX-  nUR                  SS5      R                  X7S-   S5      nU R5                  U5      nU(       a  U R7                  XU5      nU$ U R9                  U5      nU R7                  XU5      nU$ )Nr^   r   Tri   rk   r   r]   )num_prefix_tokensr&   rl   rm   )ra   rn   ro   r4   r2   rp   rq   rr   r	   r=   	unsqueezer9   r6   r`   r   r   r7   r8   rb   rs   r   r0   rx   ry   r/   rz   r;   rd   r<   )r>   rW   rg   r{   r|   rX   rY   r}   r=   r6   r7   r8   r   s                rA   r   AttentionPool2d.forward	  sm   WW
aEIIaL""1a(>>!		166!T62A6A>A		4>>00RDaHaPA*4>>+C+CA+Fbcd	M88q	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZA##Ar1dnndmmLTTUVXY[\^_abcAhhqkGA!??::1CAJJA{{2r**D<<B<'DAKK1%%aQ3IIaL

1#AHIIaLJJqQrC   )r4   r;   r   r   r   r   r   r7   r   r   r   r=   r<   r6   r9   r/   r   r8   )r   NNr   NTFr#   Fr   NNr   r   )r   r   r   r   r   r2   r   r   r   r   r   r   r   r   r   r   r-   rL   rT   r   rd   r   r   r   r   s   @rA   r   r      s@    		%%
 67*.'+&('+!!&$ %!55 S%S/125 #3-	5
  }5 sm5  }5 5 5 5 5 5 5n?4 ?S# S(3- Su||    !T ! !rC   r   )r   typingr   r   r   r2   torch.nnr0   configr   helpersr   r=   r	   pos_embed_sincosr
   r   weight_initr   Moduler   r   r+   rC   rA   <module>r      sL    * )   "  - D &N NbCbii CrC   