GjM# UdZddlZddlmZddlmZddlmZddlZddl m Z edZ edZ iZ eejjefed <ehd Zd ed ed ee e fdeee e fgee e fffdZ d9dddddejdejdejdejdzdedededejfdZedde d9dddddejdejdejdejdzdedededejfdZdedzdedefdZd ejd!ededejfd"Zd#ejd$ejd%ed&ededzd'edejfd(Zd%ed&eddfd)Zd#ejd$ejd%ed&ededzdejf d*Z d:dddddd+dd,d#ejd$ejd-ejd.ejdzd/ejdzd0ejdzd1ed2ed3ed'ededzd4ed5edzde!ejejejejffd6Z"ed7de" d:dddddd+dd,d#ejd$ejd-ejd.ejdzd/ejdzd0ejdzd1ed2ed3ed'ededzd4ed5edzde!ejejejejffd8Z#dS);zImplementations of ONNX operators as native Torch ops. NOTE: Fake implementations: Refer to https://docs.pytorch.org/docs/stable/library.html#torch.library.register_fake for more details on how to create fake kernels. N)Callable)TypeVar) ParamSpec)_dtype_mappings_P_RONNX_ATEN_DECOMP_TABLE> op_type opset_version fake_implreturncdtttfdtttfffd }|S)zDDecorator to register an ONNX operator with a custom implementation.funcrcd}tjdd|d|}|tt t tjj|<||S)Nopsetzonnx::.) mutates_args)torchlibrary custom_opr getattropsonnx register_fake)roverloadtorch_oprrrs W/var/www/html/Carbon-Document/venv/lib/python3.11/site-packages/torch/onnx/ops/_impl.py decoratorz_onnx_op..decorator's*=**=** )W ) )x ) )+    wwuy~w'G'GRRS y))))rrr)rrrr#s``` r"_onnx_opr%"sU R( Xb"f-=         r$F) interleaved num_headsrotary_embedding_dimx cos_cache sin_cache position_idsr&r'r(c*|S)zFFake implementation for RotaryEmbedding-23 for torch.compile purposes.)clone)r)r*r+r,r&r'r(s r"_rotary_embedding_23_fake_implr/5s 7799r$RotaryEmbeddingch|jt}ddtjdkfdtjjdkfdtjjdkfdtjdkodkfd nGtjd kod kfd |d krtj|d }nJ|d krDtj|dkfdd}||z} || g} tj|| }tjt|jd kd|jd } |dkr| }|ddddddd|f} |dddddd|df} |dzntjjdkojdkfdtjjdkojdkfdtjjdkfdtjjdkfdtjdtjd|r+| dddddddddf} | dddddddddf}ntj| dd\} }| z|zz }| z|zz}|r]tj|d}tj|d}tj ||fd}tj|| j} ntj ||fd} tj | | fd}|d krtj|Stj|d S)z_RotaryEmbedding-23 https://onnx.ai/onnx/operators/onnx__RotaryEmbedding.html#rotaryembedding-23rNcdjS)Nz6position_ids must be 2D when provided. Received shape shape)r,sr"z%rotary_embedding_23..Zsa\M_aar$c*ddjdS)Nz6position_ids first dim (batch) must match x.shape[0] ( ). Received rr6) batch_sizer,sr"r8z%rotary_embedding_23..^s"|Z||eqewxyez||r$r c*ddjdS)Nz;position_ids second dim (sequence) must match x.shape[-2] (r:r r6)r,sequence_lengthsr"r8z%rotary_embedding_23..bs>GRaGGo{pBCDpEGGr$c(djdjS)NzWcos_cache/sin_cache must be 2D when position_ids is provided. Received cos_cache shape , sin_cache shape r6r*r+sr"r8z%rotary_embedding_23..f-](1]]KT?]]r$c(djdjS)Nz[cos_cache/sin_cache must be 3D when position_ids is not provided. Received cos_cache shape r?r6r@sr"r8z%rotary_embedding_23..lrAr$)rr4r rBcdS)NzKnum_heads must be provided for 3D inputs. Received input tensor with shape r) input_shapesr"r8z%rotary_embedding_23..ysobmoor$cdS)Nzx should be a 4D tensor by nowrrr$r"r8z%rotary_embedding_23..s,Lr$c&djdddS)Nzcos has shape  but expected (batch=, seq=, ...)r6)r;cosr=sr"r8z%rotary_embedding_23..&jjjjjSbjjjr$c&djdddS)Nzsin has shape rIrJrKr6)r;r=sinsr"r8z%rotary_embedding_23..rMr$c,djdddS)NzLast dimension of cos cache (rP') should match rotary_embedding_dim/2 ().r6)rLrotary_embedding_dim_halfsr"r8z%rotary_embedding_23..4D " DDfDDDr$c,djdddS)NzLast dimension of sin cache (rPrRrSr6)rTrOsr"r8z%rotary_embedding_23..rUr$dim) r7lenr_checkrXpermutereshape unsqueezechunkcat)r)r*r+r,r&r'r( input_rank hidden_size head_size new_shapex_rotate x_not_rotatex1x2realimagx_rotate_concatoutputr;rLrFrTr=rOs ``` @@@@@@r"rotary_embedding_23rlCsv'K[!!JQJ!"oO     ! # a a a a      q !Z / | | | | |      q !_ 4 G G G G G     MMOOq 9Y]]__%9 ] ] ] ] ]     MMOOq 9Y]]__%9 ] ] ] ] ]   Q M!\ * * q N o o o o   "!n 9, )YG M!Y ' ' LQW"$L$LMMM Iq  (AAAqqq////0HQQQ11123334L 4 9     L ! "Fsy|'Fjjjjjj L ! "Fsy|'Fjjjjjj L " 22 D D D D D L " 22 D D D D D / Q  C / Q  C 2 aaaAAAqt!tm $ aaaAAAqt!tm $Xqb111B 8cBh D 8cBh D3tR((tR(()T4Lb999=(.AA9dD\r222 Y,/R 8 8 8FQ}V[111 = . ..r$scalerbc8||ndtj|z S)z/Get the scale factor for attention computation.Ng?)mathsqrt)rmrbs r"_get_scale_factorrqs %55C$)I2F2F,FGr$tensorr;c|jd|jd}}||z}|||||ddS)z1Reshape 3D tensor to 4D for multi-head attention.r r4)r7view transpose contiguous)rrr;r'r=rarbs r"_reshape_3d_to_4drwsS$*<?FLO[Oy(I JIFF 1a r$QKcurrent_q_num_headscurrent_kv_num_headsqk_matmul_output_modec |dkrt|||||Stjtj||ddS)z1Get QK output tensor based on the specified mode.rr3rP)_compute_qk_output_for_mode_0r zeros_likematmulru)rxryrzr{rmr|s r"_get_qk_output_for_aten_spdars[!!, q%';U    Q B0C0C D DEEEr$cJtjzdkfddS)z-Validate Group Query Attention configuration.rcdddS)Nz q_num_heads (z%) must be divisible by kv_num_heads (z ) for GQAr)r{rzsr"r8z-_validate_gqa_configuration..sy 3yyZnyyyr$N)rrZ)rzr{s``r"_validate_gqa_configurationrsA L22a7yyyyyr$c|}||kr||z}||d}t||jd}tj|}||z} ||z} t j| | ddS)zDHelper function to compute QK output for qk_matmul_output_mode == 0.r rWrBr3rP)repeat_interleaverqr7rorprrru) rxryrzr{rmK_for_qk repeat_factor scale_factor sqrt_scaleQ_scaledK_scaleds r"r~r~sH222+/CC &&}!&<<$UAGAJ77L<((J:~H*$H <("4"4R"<"< = ==r$) is_causal kv_num_heads q_num_headsr|rmsoftcapsoftmax_precisionV attn_maskpast_key past_valuerrrrrc^|jd} t|jdkrr|jd}|j}|.| ||jd|jdz|jd|zf}n| ||jd|jd|zf}|}| |||df}n|jd}|j}|A|jd|jd|jd|jdz|jdf}n|j}|}|jd|jd|jd|df}tj||j|j}tj||j|j}tj||j|j}tj||j|j}||||fS)z@Fake implementation for Attention-23 for torch.compile purposes.rrBr Nr4dtypedevice)r7rYremptyrr)rxryrrrrrrrr|rmrrr;q_sequence_length output_shapepresent_key_shapepresent_value_shapeqk_output_shaperk present_key present_value qk_outputs r"_attention_23_fake_implrs"J 17||qGAJw   q!AGAJ. l* !    l* !  0    a  GAJw     q!AGAJ. !  !" / GAJ GAJ GAJ a  [QWQX F F FF+/qwqxPPPKK 31718TTTM O1718LLLI ; y 88r$ Attentionc  d\} }}t|j}|jd}t|jdkr`tj|dko|dkd|jd}t |||}t |||}t |||}tjt|jdko/t|jdkot|jdkd|j|}t | |} |tj||g| n|}|tj||g| n|}||}}|j| }|j| }|j|}|j|}| d ko"| dko| duo|dup|jtj k}t|||rStj j ||||d || t||k }t||||| | }nY||kr3||z}||| }||| }tj|||j|j }|rutj|dud tjtj||tj |j }||t+d}|?|jtj kr%||t+d}n||z}t | |jd}t-j|} || z}!|| z}"tj|!|"dd}#|#}|#|z}$| dkr|$}| d kr| tj|$| z z}$| dkr|$}| x| t6vrX|$j}%|$t:j| }$tj|$d }&|&|%}&n-tj|$d }&ntj|$d }&| dkr|&}tj|&|}|dkr=|dd !||d}||||fS)zMAttention-23 https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23)r r4rBrrBcdS)Nz;q_num_heads and kv_num_heads must be provided for 3D inputsrrr$r"r8zattention_23..sQr$r rDcdS)Nz'Q, K, and V should be 4D tensors by nowrrr$r"r8zattention_23..s9r$NrWr)r dropout_prrm enable_gqarcdS)Nz'Cannot use both is_causal and attn_maskrrr$r"r8zattention_23..s+Tr$z-infr3rPr4)"rYr7rrZrwrqr_r.rboolrnn functionalscaled_dot_product_attentionrrzerosrtrilones masked_fillfloatrorprrutanh-_ATTENTION_23_ALLOWED_INTERMEDIATE_PRECISIONStorONNX_DTYPE_TO_TORCH_DTYPEsoftmaxrvrt)'rxryrrrrrrrr|rmrr num_head_dim sequence_dimhead_diminput_shape_lenr;r q_head_sizerrrzr{kv_sequence_length can_use_sdparkrr attn_bias causal_maskrrrrqk_matmul_output qk_with_biasoriginal_dtype qk_softmaxs' r" attention_23rcs&,3(L,!'llOJ 17||q 1  2!2 Q Q   GAJ a[ 9 9 a\ : : a\ : : L AG Ec!'lla/ECLLA4E99 '(#K e[ 1 1E    8Q-\2222 WWYY  !  :q/|4444 WWYY  qA',/7<0 -. 3 A !Q & A  % A$  ?)/UZ"?  35IJJJk-$AA #';;B   1   !    "6 6 6/3GGM##M|#DDA##M|#DDAK 1    K LT!#T#T    * %&*8 K"--{lE&MMJJI  %*,,%119*eFmmLL & 1 ) ;; Y|,, z>z>!<(2D2DR2L2LMM% ()3 A % %$I S=="UZ w0F%G%GGL A % %$I  ( $QQQ!-!3+#=>OP   #]:: "]rs/ $$$$$$'''''' ******Yt__ WT]]ACUZ2H<=BBB09 11- !$19"b&1A xB (2r6"223.)-   !    | | | ,%      \      R!?@@ )- D/  !D/D/D/ |D/|D/|D/,% D/  D/D/D/ \D/D/D/A@D/NHUT\HcHeHHHH  L &) 69  \    F |F |FF F 4< F  F \FFFF$47 > |> |>> > 4< >  \ >>>>4&*$(&* Q9!"$(Q9Q9Q9 |Q9 |Q9 |Q9|d" Q9 lT! Q9  t# Q9Q9Q9Q9Q9 4<Q9Q9TzQ9 5<u|U\ ABQ9Q9Q9Q9h +r233 &*$(&* 9!"$(999 |9 |9 |9|d" 9 lT! 9  t# 99999 4<99Tz9 5<u|U\ AB99943999r$