
    RЦi                         S SK JrJr  S SKrS SKJr  S SKJs  Jr  SSK	J
r
  SSKJr  SSKJr  SSKJr   " S S	\R"                  5      rg)
    )OptionalTypeN   )maybe_add_mask)use_fused_attn)Mlp)trunc_normal_tf_c                   V  ^  \ rS rSr% Sr\R                  R                  \   \	S'   SSSSSSSS	SS
SS\
R                  SSS4S\S\S\S\S\\   S\S\S\S\S\S\S\S\\\
R"                        S\\\
R"                        S\4U 4S jjjrS rS!S\\R(                     4S jjrS rU =r$ )"AttentionPoolLatent   z&Attention pooling w/ latent query
    
fused_attnN   g      @TFr    tokeng        in_featuresout_features	embed_dim	num_heads	feat_size	mlp_ratioqkv_biasqk_norm
latent_len
latent_dim	pos_embed	pool_type
norm_layer	act_layerdropc                   > UUS.n[         TU ]  5         U=(       d    UnU=(       d    UnX4-  S:X  d   eX@l        X4-  U l        XPl        U R                  S-  U l        Xl        [        5       U l        US:X  a6  Uc   e[        R                  " [        R                  " XQ40 UD65      U l        OS U l        U
=(       d    UU l        Xl        [        R                  " [        R                  " SU R                  U40 UD65      U l        [        R"                  " X34SU0UD6U l        [        R"                  " X3S-  4SU0UD6U l        U(       aJ  U=(       d    [        R(                  nU" U R                  40 UD6U l        U" U R                  40 UD6U l        O4[        R.                  " 5       U l        [        R.                  " 5       U l        [        R"                  " X340 UD6U l        [        R2                  " U5      U l        Ub	  U" U40 UD6O[        R.                  " 5       U l        [9        U[;        X6-  5      4SU0UD6U l        U R?                  5         g )	N)devicedtyper         absr   bias   r   ) super__init__r   head_dimr   scalepoolr   r   nn	Parametertorchzerosr   r   r   latentLinearqkv	LayerNormq_normk_normIdentityprojDropout	proj_dropnormr   intmlpinit_weights)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r!   r"   ddqk_norm_layer	__class__s                       Y/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/attention_pool.pyr(   AttentionPoolLatent.__init__   s   ( /,	#2{$)))"!."]]d*
	(*(((\\%++i*SPR*STDN!DN$1	$ll5;;q$//9#SPR#ST9EhE"E))I1}J8JrJ&6",,M'<<DK'<<DK++-DK++-DKIIi9b9	D)6@6LJ|2r2RTR]R]R_	y#i&;"<X	XUWX    c                     U R                   b.  [        U R                   U R                   R                  S   S-  S9  [        U R                  U R                  S-  S9  g )Nr   r#   )std)r   r	   shaper0   r   )r?   s    rC   r>    AttentionPoolLatent.init_weightsM   sE    >>%T^^1E1Ea1HD1PQ$//T*ABrE   	attn_maskc                    UR                   u  p4nU R                  b6  XR                  R                  S5      R                  UR                  5      -   nU R
                  R                  USS5      nU R                  U5      R                  X0R                  U R                  U R                  5      R                  SS5      nU R                  U5      R                  X4SU R                  U R                  5      R                  SSSSS5      nUR                  S5      u  pU R!                  U5      U R#                  U	5      pU R$                  (       a  [&        R(                  " XyXS9nO@XpR*                  -  nXyR                  SS5      -  n[-        X5      nUR/                  SS	9nX-  nUR                  SS5      R                  X0R                  U5      nU R1                  U5      nU R3                  U5      nXR5                  U R7                  U5      5      -   nU R8                  S
:X  a  US S 2S4   nU$ U R8                  S:X  a  UR;                  S5      nU$ )Nr   r   r&         )rJ   )dimr   avg)rH   r   	unsqueezetor"   r0   expandr2   reshaper   r   r)   	transposer3   permuteunbindr5   r6   r   Fscaled_dot_product_attentionr*   r   softmaxr8   r:   r=   r;   r+   mean)r?   xrJ   BNCq_latentr2   r3   kvattns               rC   forwardAttentionPoolLatent.forwardR   s   ''a>>%NN,,Q/22177;;A;;%%aR0FF8$$QWaabcefgWWQZaGOOPQSTVWYZ\]^yy|{{1~t{{1~1??..qQLAJJA{{2r**D!$2D<<B<'DAKK1%%a!<IIaLNN11&& 99!Q$A  YY%q	ArE   )r   r   r)   r6   r3   r0   r   r   r=   r;   r   r+   r   r8   r:   r2   r5   r*   )N)__name__
__module____qualname____firstlineno____doc__r.   jitFinalbool__annotations__r,   GELUr<   r   floatstrr   Moduler(   r>   Tensorre   __static_attributes____classcell__)rB   s   @rC   r   r      s.   		%%
 !%!'+"!!"$483577%99 9 	9
 9  }9 9 9 9 9 9 9 9 !bii19  RYY09  !9 9vC
"HU\\$: " "rE   r   )typingr   r   r.   torch.nnr,   torch.nn.functional
functionalrY   	attentionr   configr   r=   r   weight_initr	   rs   r    rE   rC   <module>r      s2    !     % "  )g")) grE   