
    RЦiY              
          S r SSKrSSKrSSKJrJr  SSKrSSKJr  SSK	Js  J
r  SSKJr  SSKJr  SSKJr  SSKJr  \" \R*                  R-                  S	S5      5      S:  r   S(S
\\\4   S\\\\4      S\S\R2                  4S jjrS\\\4   S\\S4   4S jr  S)S\S\4S jjrS\\\4   S\\S4   4S jr " S S\R>                  5      r     S*S\\\4   S\\\4   4S jjr! " S S\R>                  5      r"   S+S\S\\   4S  jjr#S!\S"\S#\R2                  S$\R2                  S\R2                  4
S% jr$ " S& S'\R>                  5      r%g),zeRelative position embedding modules and functions

Hacked together by / Copyright 2022 Ross Wightman
    N)OptionalTuple   )ndgrid)RegularGridInterpolator)Mlp)trunc_normal_TIMM_USE_SCIPY_INTERPq_sizek_sizeclass_tokenreturnc           
         Ub   S5       e[         R                  " [        [         R                  " U S   US9[         R                  " U S   US95      5      R	                  S5      nUS S 2S S 2S 4   US S 2S S S 24   -
  nUR                  SSS5      nUS S 2S S 2S4==   U S   S-
  -  ss'   US S 2S S 2S4==   U S   S-
  -  ss'   US S 2S S 2S4==   SU S   -  S-
  -  ss'   SU S   -  S-
  SU S   -  S-
  -  nUR                  S5      nU(       a5  [        R                  " U/ SQ5      nXgSSS 24'   US-   USS 2S4'   US-   US'   UR                  5       $ )	Nz-Different q & k sizes not currently supportedr   devicer      )r   r   r   r   r   r   )
torchstackr   arangeflattenpermutesumFpad
contiguous)r   r   r   r   coordsrelative_coordsnum_relative_distancerelative_position_indexs           X/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/pos_embed_rel.pygen_relative_position_indexr#      s    >JJJ>[[VAYv.VAYv.  wqz  Q4Z(6!T1*+==O%--aA6OAq!Gq	A-Aq!Gq	A-Aq!GF1I 11]Q.1vay=13DE0 .11"5 #$%%(?"N)>12&)>)BA&(=(A%"--//    new_window_sizenew_bias_shape.c           	      \   US   S-  S-
  US   S-  S-
  4nU R                   S:X  az  Uu  pEnU R                  u  pxn	XSS   :X  a  XcS   :X  d   eX:w  d  X:w  aH  [        R                  R                  R                  U R                  S5      USSS9R                  S5      n U $ U R                   S:X  d   eUu  pU R                  u  pXS   US   -  -
  n[        X-
  S-  5      nX4nUS   US   :w  d  US   US   :w  a  U(       a  X* S 2S S 24   nU S U* 2S S 24   n OS n[        R                  R                  R                  U R                  SS5      R                  SS	US   US   45      USSS9R                  S	X-
  5      R                  SS5      n Ub  [        R                  " X4SS
9n U $ )Nr   r   r      bicubicF)sizemodealign_corners      ?r   dim)ndimshaper   nn
functionalinterpolate	unsqueezesqueezeint	transposereshapeviewcat)rel_pos_biasr%   r&   dst_size_dst_hdst_wnum_attn_headssrc_hsrc_wdst_num_possrc_num_posnum_extra_tokenssrc_sizeextra_tokenss                  r"    resize_rel_pos_bias_table_simplerI   M   s   
  "Q&*OA,>,BQ,FGHA(%'3'9'9$u#!(<<<>U^ 88..::&&q)#	 ; 
 gaj @ 3   A%%%'&2&8&8#&1+*CD63>?'A;(1+%!)C+,=,>,AB+,>.>->,>,AB# 88..::&&q!,44aXa[(ST+5VW#	 ; 
 d2{56yyA  '$yy,)E1Mr$   interpolation	antialiasc                    U R                  5       u  pEUu  pgXW:X  d   eXF:w  a  U R                  nU R                  5       n [        US-  5      n	[        US-  5      n
[        R
                  " U R                  SS5      R                  SXYU	5      X4UUS9nUR                  Xv5      R                  SS5      nUR                  U5        U$ U $ )z
Resample relative position bias table suggested in LeVit
Adapted from: https://github.com/microsoft/Cream/blob/main/TinyViT/utils.py
r-   r   r   )r*   r+   rK   )	r*   dtypefloatr7   r   r4   r   r:   to)position_bias_tablenew_sizerJ   rK   L1nH1L2nH2
orig_dtypeS1S2$relative_position_bias_table_resizeds               r"   resize_rel_pos_bias_table_levitrZ   |   s     "&&(GBGB::	x(..
1779s^s^/0}}''1-221crB	0
, 0T/X/XY\/a/i/ijkmn/o,,//
;33""r$   c                 >  ^ [         (       a  SSKJn  US   S-  S-
  US   S-  S-
  4nU R                  S:X  a.  SnUu  pgnXtS   :X  a  XS   :X  d   eU R                  u  pnX4nSnOFU R                  S:X  d   eUu  pU R                  u  pXS   US   -  -
  n[        X-
  S-  5      nX4nSnUS   US   :w  d  US   US   :w  GaW  U(       a  X* S	2S	S	24   nU S	U* 2S	S	24   n OS	nS
 mU4S jnU" US   US   5      nU" US   US   5      n[        R                  " U5      [        R                  " U5      /nUS   S-  nUS   S-  n[        R                  " U* US-   S5      n[        R                  " U* US-   S5      n[        UU5      n/ n[        U	5       GH(  nU(       a.  U S	S	2U4   R                  US   US   5      R                  5       nOU US	S	2S	S	24   R                  5       n[         (       ae  WR                  UUUR                  5       SS9n[        R                  " U" UU5      5      R!                  5       R#                  U R$                  5      nO;['        UU5      nU" U5      R!                  5       R#                  U R$                  5      nU(       a  UR                  SS5      nUR)                  U5        GM+     U(       a  [        R*                  " USS9n O[        R*                  " USS9n Ub   U(       d   e[        R*                  " U U4SS9n U $ )ao  Resize relative position bias table using more advanced interpolation.

Modified from code in Microsoft Unilm (https://github.com/microsoft/unilm) repo (BeiT, BeiT-v2, etc).

https://github.com/microsoft/unilm/blob/5255d52de86dad642810f5849dd357769346c1d7/beit/run_class_finetuning.py#L351

Args:
    rel_pos_bias:
    new_window_size:
    new_bias_shape:

Returns:

r   )r4   r   r   r(   Fr-   TNc                 "    U SX-  -
  -  SU-
  -  $ )N      ? )arns      r"   geometric_progression8resize_rel_pos_bias_table.<locals>.geometric_progression   s    af%q11r$   c                 ,  > Su  p#X2-
  S:  a*  X#-   S-  nT" SX@S-  5      nXQS-  :  a  UnOUnX2-
  S:  a  M*  / nSn[        U S-  5       H  nUR                  U5        UWUS-   -  -  nM!     [        U5       V	s/ s H  o* PM     n
n	U
S/-   U-   $ s  sn	f )N)g)\(?g      ?gư>       @r   r   r   )rangeappendreversed)srcdstleftrightqgpdiscurir>   r_idsrb   s              r"   _calc(resize_rel_pos_bias_table.<locals>._calc   s    #KD,%\S(*1a:q=ED ,% CC3!8_

3qQU|# % "*#/ARE/A3;$$ 0s   :Bre   g?r]   cubic)kindr   r.   )
_USE_SCIPYscipyr4   r0   r1   r7   r   tensorr   r   rf   r:   rN   interp2dnumpyTensorr   rO   r   r   rg   r;   ) r<   r%   r&   r4   r=   rF   r>   r?   r@   rA   rB   rC   rG   has_flat_shaperD   rE   rH   rs   yxyxtytxdydxdyxall_rel_pos_biasrq   zfr`   rb   s                                   @r"   resize_rel_pos_bias_tabler      s1   & z%"Q&*OA,>,BQ,FGHA(%#!(<<<'3'9'9$u>  A%%%'&2&8&8#&1+*CD63>?'{hqk!Xa[HQK%?'(9(:A(=>L'(:*:):(:A(=>LL	2	%$ (1+x{+(1+x{+ll1ou||A/ a[Ca[C\\2#rCx-\\2#rCx-Rn ~&A A&++HQK!EKKM Aq)//1z((Aqwwyw(GLL2r+668;;L<O<OP
 ,B2cF%%'**<+>+>?FF2qM##A&' '*  99%52>L 99%51=L#!!> 99lL%AqILr$   c                      ^  \ rS rSrSr   SS\\\4   S\S\4U 4S jjjrSS jrSS	 jr	S\
R                  4S
 jrSS\\
R                     4S jjrSS jrSrU =r$ )
RelPosBiasi  zVRelative Position Bias
Adapted from Swin-V1 relative position bias impl, modularized.
window_size	num_headsprefix_tokensc           	        > XES.n[         T	U ]  5         US::  d   eXl        US   US   -  U l        X0l        U R                  U-   4S-  U4-   U l        SUS   -  S-
  SUS   -  S-
  -  SU-  -   n[        R                  " [        R                  " Xr40 UD65      U l
        U R                  U-   S-  nU R                  S[        R                  " X[        R                  S9SS9  U R                  5         g )	Nr   rM   r   r   r   r(   r!   F
persistent)super__init__r   window_arear   
bias_shaper2   	Parameterr   emptyrelative_position_bias_tableregister_bufferlongreset_parameters)
selfr   r   r   r   rM   ddr    
index_size	__class__s
            r"   r   RelPosBias.__init__  s    /!!!&&q>KN:*++m;=AYLP!"[^!3a!7AA<NQR<R SVWZgVg g,.LLEZ9lik9l,m)&&61<
%KK
D 	 	
 	r$   r   c                 L    [        U R                  SS9  U R                  5         g"Initialize parameters and buffers.g{Gz?)stdN)r	   r   _init_buffersr   s    r"   r   RelPosBias.reset_parameters1  s    d77SAr$   c                     U R                   R                  [        U R                  U R                  S:  U R                   R
                  S9R                  S5      5        g).Compute and fill non-persistent buffer values.r   )r   r   r   N)r!   copy_r#   r   r   r   r:   r   s    r"   r   RelPosBias._init_buffers6  sO    $$**'   ..233:: d2h	
r$   c                     U R                   U R                     nUR                  U R                  5      R	                  SSS5      nUR                  S5      R                  5       $ )Nr   r   r   )r   r!   r:   r   r   r5   r   r   relative_position_biass     r"   get_biasRelPosBias.get_bias@  sZ    !%!B!B4C_C_!`!7!<!<T__!M!U!UVWYZ\]!^%//2==??r$   shared_rel_posc                 &    XR                  5       -   $ Nr   r   attnr   s      r"   forwardRelPosBias.forwardF      mmo%%r$   c                 $    U R                  5         gz"Initialize non-persistent buffers.Nr   r   s    r"   init_non_persistent_buffers&RelPosBias.init_non_persistent_buffersI      r$   )r   r   r   r   r   r   NNr   Nr   __name__
__module____qualname____firstlineno____doc__r   r7   r   r   r   r   r|   r   r   r   r   __static_attributes____classcell__r   s   @r"   r   r     s|     "# sCx    	   8

@%,, @&HU\\,B & r$   r   win_sizepretrained_win_sizec                    US;   d   e[         R                  " U S   S-
  * U S   US9R                  [         R                  5      n[         R                  " U S   S-
  * U S   US9R                  [         R                  5      n[         R                  " [        XV5      5      nUR                  SSS5      R                  5       nUS:X  a  US   S:  a5  US S 2S S 2S4==   US   S-
  -  ss'   US S 2S S 2S4==   US   S-
  -  ss'   O4US S 2S S 2S4==   U S   S-
  -  ss'   US S 2S S 2S4==   U S   S-
  -  ss'   US-  n[         R                  " U5      [         R                  " SUR                  5       -   5      -  [        R                  " S5      -  nO>[         R                  " U5      [         R                  " SUR                  5       -   5      -  nUR                  U5      $ )	N)swincrr   r   r   r   r      r]   )r   r   rO   float32r   r   r   r   signlog2absmathlog)r   r   r+   r   rM   relative_coords_hrelative_coords_wrelative_coords_tables           r"   gen_relative_log_coordsr   N  s    >!!!x{Q%7!VTWWX]XeXefx{Q%7!VTWWX]XeXef!KK/@(TU199!QBMMOv~q!A%!!Q'*/B1/E/IJ*!!Q'*/B1/E/IJ*!!Q'*x{Q?*!!Q'*x{Q?*" %

+@ AEJJ'++--E/ !/151!> !&

+@ AEII'++--E/ !/ !##E**r$   c                      ^  \ rS rSrSr       SS\\\4   S\S\S\S\S\\\4   4U 4S	 jjjrS
\	R                  4S jrSS\\	R                     4S jjrSS jrSS jrSS jrSrU =r$ )	RelPosMlpim  zLog-Coordinate Relative Position MLP
Based on ideas presented in Swin-V2 paper (https://arxiv.org/abs/2111.09883)

This impl covers the 'swin' implementation as well as two timm specific modes ('cr', and 'rw')
r   r   
hidden_dimr   r+   pretrained_window_sizec	           	        > XxS.n	[         TU ]  5         Xl        U R                  S   U R                  S   -  U l        X@l        X l        U R                  4S-  U4-   U l        XPl        X`l        US:X  a$  [        R                  " 5       U l        SU l        Sn
O#[        R                  " 5       U l        S U l        Sn
[         SUU[        R                  U
S	S
.U	D6U l        U R                  S-  nSUS   -  S-
  SUS   -  S-
  S4nU R#                  S[$        R&                  " X[$        R(                  S9SS9  U R#                  S[$        R&                  " U40 U	D6SS9  U R+                  5         g )Nr   r   r   r   r      )TFT)g      ?g        )hidden_featuresout_features	act_layerbiasdropr!   Fr   rel_coords_log)r   )r   r   r   r   r   r   r   r+   r   r2   Sigmoidbias_act	bias_gainIdentityr   ReLUmlpr   r   r   r   r   )r   r   r   r   r   r+   r   r   rM   r   mlp_biasr   rel_coords_shaper   s                r"   r   RelPosMlp.__init__s  s~    /&++A.1A1A!1DD*"++-1YL@	&<#6>JJLDMDN$HKKMDM!DNH
&"gg
 
 %%*
A.2AA4F4JAN%KK
D 	 	

 	KK(/B/ 	 	
 	r$   r   c                    U R                  U R                  5      nU R                  bD  UR                  SU R                  5      U R                     nUR                  U R
                  5      nUR                  SSS5      nU R                  U5      nU R                  b  U R                  U-  nU R                  (       a.  [        R                  " XR                  SU R                  S/5      nUR                  S5      R                  5       $ )Nr   r   r   r   )r   r   r!   r:   r   r   r   r   r   r   r   r   r5   r   r   s     r"   r   RelPosMlp.get_bias  s    !%$*=*=!>''3%;%@%@T^^%TUYUqUq%r"%;%@%@%Q"!7!?!?1a!H!%/E!F>>%%)^^6L%L"%&UU+ADVDVXY[_[m[mopCq%r"%//2==??r$   r   c                 &    XR                  5       -   $ r   r   r   s      r"   r   RelPosMlp.forward  r   r$   c                 $    U R                  5         g)r   Nr   r   s    r"   r   RelPosMlp.reset_parameters  r   r$   c           
      \   U R                   R                  nU R                  R                  nU R                   R	                  [        U R                  US9R                  S5      5        U R                  R	                  [        U R                  U R                  U R                  UUS95        g)r   r   r   )r+   r   rM   N)r!   r   r   rM   r   r#   r   r:   r   r   r+   r   r   rM   s      r"   r   RelPosMlp._init_buffers  s    --44##))$$**'(8(8HMMbQ	
 	!!#  ++YY	
r$   c                 $    U R                  5         gr   r   r   s    r"   r   %RelPosMlp.init_non_persistent_buffers  r   r$   )
r   r   r   r   r+   r   r   r   r   r   )r      r   r   r   NNr   r   )r   r   r   r   r   r   r7   strr   r   r|   r   r   r   r   r   r   r   r   r   s   @r"   r   r   m  s     !!"6<5 sCx5  5  	5 
 5  5  %*#s(O5  5 n@%,, @&HU\\,B &
" r$   r   lengthmax_relative_positionc                     Uc  U S-
  nSU-  S-   n[         R                  " X XBUS9n[        U 5       H5  n[        U 5       H#  nXv-
  U-   n[        Xv-
  5      U:  a  M  SXVXx4'   M%     M7     U$ )a  Generate a one_hot lookup tensor to reindex embeddings along one dimension.

Args:
    length: the length to reindex to.
    max_relative_position: the maximum relative position to consider.
        Relative position embeddings for distances above this threshold
        are zeroed out.
Returns:
    a lookup Tensor of size [length, length, vocab_size] that satisfies
        ret[n,m,v] = 1{m - n + max_relative_position = v}.
r   r   r   )r   zerosrf   r   )	r   r   r   rM   
vocab_sizeretrq   r   vs	            r"   generate_lookup_tensorr    s    " $ &
**Q.J
++fju
MC6]vA--A15z11C1L	   Jr$   heightwidthheight_lookupwidth_lookupc                     [         R                  " SX5      n[         R                  " SXT5      nX-  nUR                  U R                  S   Xf5      $ )a(  Reindex 2d relative position bias with 2 independent einsum lookups.

Adapted from:
 https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py

Args:
    relative_position_tensor: tensor of shape
        [..., vocab_height, vocab_width, ...].
    height: height to reindex to.
    width: width to reindex to.
    height_lookup: one-hot height lookup
    width_lookup: one-hot width lookup
Returns:
    reindexed_tensor: a Tensor of shape
        [..., height * width, height * width, ...]
znhw,ixh->nixwznixw,jyw->nijxyr   )r   einsumr9   r1   )relative_position_tensorr  r  r  r	  reindexed_tensorareas          r"   reindex_2d_einsum_lookupr    sO    . ||O5M]||$57GV>D##$<$B$B1$EtRRr$   c                      ^  \ rS rSrSr   SS\\\4   S\S\4U 4S jjjrSS jrSS	 jr	S\
R                  4S
 jrSS\\
R                     4S jjrSS jrSrU =r$ )RelPosBiasTfi  zRelative Position Bias Impl (Compatible with Tensorflow MaxViT models)
Adapted from:
 https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py
r   r   r   c                   > XES.n[         TU ]  5         US::  d   eXl        US   US   -  U l        X l        SUS   -  S-
  nSUS   -  S-
  nU R                  Xx4U l        [        R                  " [        R                  " U R
                  40 UD65      U l
        US   US   U4n	US   US   U4n
U R                  S[        R                  " U	40 UD6SS9  U R                  S[        R                  " U
40 UD6SS9  U R                  5         g )	Nr   r   r   r   r  Fr   r	  )r   r   r   r   r   r   r2   r   r   r   r   r   r   )r   r   r   r   r   rM   r   vocab_heightvocab_widthheight_lookup_shapewidth_lookup_shaper   s              r"   r   RelPosBiasTf.__init__  s#    /!!!&&q>KN:";q>)A-+a.(1,>><E,.LLT__9[XZ9[,\)*1~{1~|L)!nk!nkJ_ekk:M.TQS.Tafg^U[[9K-Rr-R_de 	r$   r   c                 t    [         R                  R                  U R                  SS9  U R	                  5         gr   )r2   initnormal_r   r   r   s    r"   r   RelPosBiasTf.reset_parameters0  s(    
99sCr$   c                    U R                   R                  nU R                   R                  nU R                   R                  [	        U R
                  S   XS95        U R                  R                  [	        U R
                  S   XS95        g)r   r   r   r   N)r  r   rM   r   r  r   r	  r   s      r"   r   RelPosBiasTf._init_buffers5  st    ##**""((  !78H8H8KTZ!hi 6t7G7G7JSY ghr$   c                     [        U R                  U R                  S   U R                  S   U R                  U R                  5      $ )Nr   r   )r  r   r   r  r	  r   s    r"   r   RelPosBiasTf.get_bias<  sF    '--QQ
 	
r$   r   c                 &    XR                  5       -   $ r   r   r   s      r"   r   RelPosBiasTf.forwardF  r   r$   c                 $    U R                  5         gr   r   r   s    r"   r   (RelPosBiasTf.init_non_persistent_buffersI  r   r$   )r   r   r   r   r   r   r   r   r   r   s   @r"   r  r    s{     "# sCx    	   6
i
%,, 
&HU\\,B & r$   r  )NFN)r)   T)r   r   NN)NNN)&r   r   ostypingr   r   r   torch.nnr2   torch.nn.functionalr3   r   gridr   r4   r   r   r   weight_initr	   r7   environgetrw   boolr|   r#   rI   r   rZ   r   Moduler   r   r   r  r  r  r^   r$   r"   <module>r.     s    	 "      0  & 7;<q@

 -1!	50c3h50sCx)50 50
 \\50p,sCx, c3h,d '	# # 	#@qsCxq c3hqh; ;@ 06+S/+"38_+>d		 dR 04	'}>SS S ||	S
 llS \\S:;299 ;r$   