
    ϑiK&                    :   S SK Jr  S SKJrJr  S SKJr  S SKJr  S SK	r	\(       a  S SK
Jr   " S S\5      r " S	 S
\5      r " S S\5      r\ " S S5      5       r\ " S S\5      5       r\ " S S\5      5       r\ " S S\5      5       r\ " S S\5      5       rg)    )annotations)ABCabstractmethod)	dataclass)TYPE_CHECKINGN)Sequencec                  >    \ rS rSr\\R                  4S j5       rSrg)AttentionBias#   c                    [         eNNotImplementedErrorselfshapedtypes      \/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/incubate/nn/attn_bias.pymaterializeAttentionBias.materialize$   s    !!     N)	__name__
__module____qualname____firstlineno__r   paddlefloat32r   __static_attributes__r   r   r   r
   r
   #   s    '-~~ " "r   r
   c                  :    \ rS rSr\R
                  4S jrS rSrg)LowerTriangularMask)   c                    U[         R                  La  UO[         R                  n[         R                  " U[	        S5      US9n[         R
                  " USS9R                  U5      $ )N-inf)r   
fill_valuer      diagonalr   bfloat16r   fullfloattriuastype)r   r   r   	create_astensors        r   r   LowerTriangularMask.materialize*   sO    "&//9Ev~~	E&M
 {{6A.55e<<r   c                    [        U5      $ r   )!LowerTriangularMaskWithTensorBiasr   biass     r   add_biasLowerTriangularMask.add_bias1   s    066r   r   N)	r   r   r   r   r   r   r   r6   r   r   r   r   r!   r!   )   s    '-~~ =7r   r!   c                  H   ^  \ rS rSrS r\R                  4U 4S jjrSrU =r	$ )r3   5   c                    Xl         g r   _biasr4   s     r   __init__*LowerTriangularMaskWithTensorBias.__init__6   s    
r   c                <   > [         TU ]  X5      U R                  -   $ r   )superr   r<   )r   r   r   	__class__s      r   r   -LowerTriangularMaskWithTensorBias.materialize9   s    w"504::==r   r;   )
r   r   r   r   r=   r   r   r   r   __classcell__rA   s   @r   r3   r3   5   s     (.~~ > >r   r3   c                  T    \ rS rSr% S\S'   S\S'   S\S'   S r\S	 5       rSS jrSr	g
)
SeqLenInfo=   paddle.Tensorseqstartint
max_seqlenz	list[int]seqstart_pyc              #  f   #    [        U R                  U R                  SS  5       S h  vN   g  N7f)Nr&   )ziprL   r   s    r   	intervalsSeqLenInfo.intervalsC   s(     t'')9)9!")=>>>s   '1/1c                    S/nSnU H%  n[        X45      nUR                  US   U-   5        M'     [        R                  " U[        R                  S9nU " X5US9$ )Nr   r   )rK   rI   rL   )maxappendr   	to_tensorint32)clsseqlensrL   rK   seqlenrI   s         r   from_seqlensSeqLenInfo.from_seqlensF   sc    c
FZ0J{278  ##Kv||D!+
 	
r   Nc           
        U R                   S   UR                  S   :X  a  UR                  S   S:X  d   eUc  S/[        U R                   5      S-
  -  n/ nSnU H7  nUR                  U R                   XE-      U R                   U   -
  5        XE-  nM9     [	        X!R                  USS95       VVs/ s H'  u  pgUR                  US/UR                  SS  Q5      PM)     snn$ s  snnf )NrS   r&   r   axis   )rL   r   lenrV   rN   splitreshape)r   xbatch_sizessplit_chunksit
batch_sizebsr0   s           r   rc   SeqLenInfo.splitR   s    #qwwqz1aggajAoEE#T%5%5!6!:;K%J  1D4D4DR4HH B	 & "+ww|!w/LM
M
 NNB6V\\!"%567M
 	
 
s   2.C$r   r   )
r   r   r   r   __annotations__rP   classmethodr\   rc   r   r   r   r   rF   rF   =   s1    O? 	
 	

r   rF   c                  h   ^  \ rS rSr% S\S'   S\S'   U 4S jr\S 5       r\S 5       rSS	 jr	S
r
U =r$ )PaddedSeqLenInfoc   rH   r[   zSequence[int]	seqlen_pyc              #  z   >#    [        [        TU ]	  5       U R                  5       H  u  u  pnXU-   4v   M     g 7fr   )rN   r@   rP   rq   )r   start_lengthrA   s       r   rP   PaddedSeqLenInfo.intervalsh   s8     "%eg&7&94>>"JJU'' #Ks   8;c                    [        S5      e)NzQPlease use SeqLenInfo.from_seq_lens() or PaddedSeqLenInfo.from_seq_lens_padded().r   )rY   rZ   s     r   r\   PaddedSeqLenInfo.from_seqlensl   s    !_
 	
r   c           
     0  ^ [        U4S jU 5       5      (       d   e[        [        S[        U5      T-  S-   T5      5      nU " [        R
                  " U[        R                  S9U[        U5      [        R
                  " U[        R                  S9US9$ )Nc              3  ,   >#    U  H	  oT:*  v   M     g 7fr   r   ).0r[   paddings     r   	<genexpr>7PaddedSeqLenInfo.from_seqlens_padded.<locals>.<genexpr>t   s     ;7W$7s   r   r&   rT   )r[   rq   rK   rI   rL   )alllistrangerb   r   rW   rX   rU   )rY   rZ   r|   rL   s     ` r   from_seqlens_padded$PaddedSeqLenInfo.from_seqlens_paddedr   s}    ;7;;;;;5CL7$:Q$>HI##G6<<@7|%%kF#
 	
r   c                    [         er   r   )r   re   rf   s      r   rc   PaddedSeqLenInfo.split~   s    !!r   r   r   )r   r   r   r   rl   rP   rm   r\   r   rc   r   rC   rD   s   @r   ro   ro   c   sF    ( 
 

 	
 	
" "r   ro   c                      \ rS rSr% S\S'   S\S'   SrS\S'   \R                  4S jr\R                  4S	 jr	\
SS
 j5       r\
S 5       r\
SS j5       rS rS rS rS rSrg)BlockDiagonalMask   rF   	q_seqinfo	k_seqinfoNzSequence[int] | None_batch_sizesc                *    [         R                  " XS9$ N)r   r   )r   zerosr   s      r   _create_block_mask$BlockDiagonalMask._create_block_mask   s    ||%55r   c                "   US   U R                   R                  S   :X  d   eUS   U R                  R                  S   :X  d   e[        R                  " USS  [        S5      US9n[        U R                  R                  5       U R                   R                  5       5       H*  u  u  pEu  pgXT-
  Xv-
  /nU R                  X5      X4U2Xg24'   M,     [        [        U5      S-
  5       H  n	UR                  S5      nM     UR                  U5      $ )NrS   r$   )r%   r   ra   r   )r   rL   r   r   r+   r,   rN   rP   r   r   rb   	unsqueezeexpand)
r   r   r   maskq_startq_endk_startk_end	sub_shapert   s
             r   r   BlockDiagonalMask.materialize   s    RyDNN66r::::RyDNN66r::::{{5:%-uM25NN$$&(@(@(B3
.W.w %/:I151H1H2D-.	3
 s5zA~&A>>!$D '{{5!!r   c                    Ub  [        U5      [        U5      :X  d   e[        R                  U5      nUb  X:X  a  UnO[        R                  U5      nU " X4S9$ )N)r   r   )rb   rF   r\   )rY   q_seqlen	kv_seqlenr   r   s        r   r\   BlockDiagonalMask.from_seqlens   sX     CMS^$CCC++H5	 5!I"//	:IY<<r   c                   U Vs/ s H  o"R                   S   PM     nn/ nU H@  n[        UR                   S   5       H!  nUR                  UR                   S   5        M#     MB     U R                  U5      nX7l        [
        R                  " U Vs/ s H$  oUR                  SS/UR                   SS  Q5      PM&     snSS9nXx4$ s  snf s  snf )Nr   r&   rS   ra   r_   )r   r   rV   r\   r   r   concatrd   )	rY   tensorsr0   rf   rZ   re   rt   
block_diagconcated_tensors	            r   from_tensor_list"BlockDiagonalMask.from_tensor_list   s    5<=W6||AW=A1771:&qwwqz* '  %%g.
"- --7>?w!YY2,,-w?a
 ** > @s   C+Cc                   [        U5      [        U5      :X  d   eUb  [        U5      [        U5      :X  d   eU Vs/ s H  oDR                  S   PM     nn/ / pv[        [        X5      5       H  u  nu  pU	R                  S   U
R                  S   :X  d   eUR	                  U	R                  S   /U	R                  S   -  5        UR	                  U
R                  S   /U
R                  S   -  5        Uc  M  X8   R                  S S U
R                  S S :X  a  M   e   U R                  Xg5      nU Vs/ s H  oR                  S   PM     snUl        U[        R                  " U Vs/ s H$  oR                  SS/UR                  SS  Q5      PM&     snSS9[        R                  " U Vs/ s H$  oR                  SS/UR                  SS  Q5      PM&     snSS9UbF  [        R                  " U Vs/ s H$  oR                  SS/UR                  SS  Q5      PM&     snSS94$ S 4$ s  snf s  snf s  snf s  snf s  snf )Nr   r&   ra   rS   r_   )
rb   r   	enumeraterN   extendr\   r   r   r   rd   )rY   	tensors_q	tensors_k	tensors_vr0   rf   	q_seqlens
kv_seqlensiqkr   re   s                r   from_tensor_lists_qkv'BlockDiagonalMask.from_tensor_lists_qkv   s)   9~Y/// C	Nc)n$DDD5>?Y6||AY? "B:"3y#<=IAv771:+++aggaj\AGGAJ67qwwqzlQWWQZ78$	(:(:2A(>!''"1+(MMM	 >
 %%i<
7@"Ay!771:y"A
MM;DE9aAr0AGGABK019EA MM;DE9aAr0AGGABK019EA (	 ?HIy!YY2445yI
 	
 
 	
 @ #B F F Js   H,7H1,+H6
0+H;7+I c                L    U R                   R                  XR                  5      $ r   )r   rc   r   r   r0   s     r   split_queriesBlockDiagonalMask.split_queries       ~~##F,=,=>>r   c                L    U R                   R                  XR                  5      $ r   )r   rc   r   r   s     r   split_kvBlockDiagonalMask.split_kv   r   r   c                    U R                   U R                  L d   eU R                   R                  XR                  5      $ r   )r   r   rc   r   r   s     r   rc   BlockDiagonalMask.split   s3    ~~///~~##F,=,=>>r   c                T    [        U R                  U R                  U R                  S9$ )N)r   r   r   )BlockDiagonalCausalMaskr   r   r   rO   s    r   make_causalBlockDiagonalMask.make_causal   s&    &nnnn**
 	
r   r   r   )r   r   r   r   rl   r   r   r   r   r   rm   r\   r   r   r   r   rc   r   r   r   r   r   r   r      s    )-L&-.4nn 6 (.~~ " = = + + 
 
<???
r   r   c                  4    \ rS rSr\R
                  4S jrSrg)r      c                0    [        5       R                  XS9$ r   )r!   r   r   s      r   r   *BlockDiagonalCausalMask._create_block_mask   s    "$00u0JJr   r   N)r   r   r   r   r   r   r   r   r   r   r   r   r      s    .4nn Kr   r   c                      \ rS rSr% S\S'   S\S'   SrS\S'   S	\R                  4S
 jr\R                  4S jr	\
 SS j5       rSrg)+BlockDiagonalCausalWithOffsetPaddedKeysMask   rF   r   ro   r   Nzpaddle.Tensor | Nonecausal_diagonalr   c                    U[         R                  La  UO[         R                  n[         R                  " X[	        S5      S9n[         R
                  " USU-   S9R                  U5      $ )Nr$   r   r%   r&   r'   r)   )r   r   offsetr   r/   r0   s         r   r   >BlockDiagonalCausalWithOffsetPaddedKeysMask._create_block_mask   sL    "&//9Ev~~	UfN{{6AJ7>>uEEr   c                   US   U R                   R                  S   :X  d   eUS   U R                  R                  S   :X  d   e[        R                  " USS  U[        S5      S9n[        [        U R                  R                  5       U R                   R                  5       5      5       H^  u  nu  u  pVu  pxU R                  Xe-
  X-
  4U R                  c  SO%[        U R                  U   R                  5       5      US9X5U2Xx24'   M`     [        [        U5      S-
  5       H  n	UR                  S5      nM     UR!                  U5      $ )NrS   r   r$   r   r   )r   r   ra   )r   rL   r   r   r+   r,   r   rN   rP   r   r   rJ   itemr   rb   r   r   )
r   r   r   r   r   r   r   r   r   rt   s
             r   r   7BlockDiagonalCausalWithOffsetPaddedKeysMask.materialize   s=   RyDNN66r::::RyDNN66r::::{{5:UuV}M7@((*DNN,D,D,FG8
3A3 "27 261H1H%/2 ++3 T11!499;< 2I 2D-.8
 s5zA~&A>>!$D '{{5!!r   c                    Ub  [        U5      [        U5      :X  d   e[        R                  U5      n[        R	                  X25      nU " UUUS9$ )N)r   r   r   )rb   rF   r\   ro   r   )rY   r   
kv_paddingr   r   r   r   s          r   r\   8BlockDiagonalCausalWithOffsetPaddedKeysMask.from_seqlens  sU      CMS^$CCC++H5	$88O	+
 	
r   r   r   )r   r   r   r   rl   r   r   r   r   r   rm   r\   r   r   r   r   r   r      sM    ,0O)0/0 F
 (.~~ "( >B

 

r   r   )
__future__r   abcr   r   dataclassesr   typingr   r   collections.abcr   r
   r!   r3   rF   ro   r   r   r   r   r   r   <module>r      s   . # # !   ("C "	7- 	7>(; > "
 "
 "
J "z " "< ]
 ]
 ]
@ K/ K K
 )
- )
 )
r   