GjA9 UdZddlmZddlZddlmZddlmZddlm Z m Z ddl m Z m Z dd lmZe rdd lmZddlZdd lmZd gZdad ed<eGddZed`dZ dadbdZedcdZdddZ dedfd'Z dgdhd3Zdid7Zdjd9Z e d:Z!dkd=Z" dldmdHZ# dndodLZ$ddddddddMddN dpdQZ%ddddddddRdqdSZ&ddddTdrdWZ' dsddYdtdZZ(ddYdud]Z)ej*d^e_dS)vzUBER PROTOTYPE!!!) annotationsN) dataclass)cache)Any TYPE_CHECKING) TypeVarTupleUnpack) _registry) ModuleType)Libraryregister_flash_attention_fa4 str | None_FA4_MODULE_PATHc"eZdZUded<ddZdS) _FA4HandlezLibrary | NonelibraryreturnNonecd|_dSN)r)selfs Z/var/www/html/Carbon-Document/venv/lib/python3.11/site-packages/torch/nn/attention/_fa4.pyremovez_FA4Handle.remove"s  N)rr)__name__ __module__ __qualname____annotations__rrrrrs6rrdevice torch.devicerintcJtj|\}}|Sr)torchcudaget_device_capability)r!major_s r_get_device_majorr*&s z//77HE1 Lrflash_attn.cute.interface module_pathstrcZt|}|attS)z Register FA4 flash attention kernels with the PyTorch dispatcher. Args: module_path: Python module path to the FA4 implementation. )_fa4_import_modulerr_fa4_register_kernels)r,r)s rrr,s, ;''A" +-- . ..rr ctj|}t|drt|dstd|d|S)N_flash_attn_fwd_flash_attn_bwdzModule 'z' does not expose FA4 kernels) importlib import_modulehasattr RuntimeError)r,modules rr/r/;sX  $[ 1 1F 6, - -RWVEV5W5WRPkPPPQQQ Mrr c@tddd}|dtd|dtd|dtd|dt d|dt d|S) NatenIMPLCUDA_flash_attention_forward+_flash_attention_forward_no_dropout_inplace_flash_attention_backward#_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward)r impl!_fa4_flash_attention_forward_impl4_fa4_flash_attention_forward_no_dropout_inplace_impl"_fa4_flash_attention_backward_impl4_fa4_scaled_dot_product_flash_attention_forward_impl5_fa4_scaled_dot_product_flash_attention_backward_impl)libs rr0r0Cs &&& ) )CHH ')JFSSSHH5< HH (*LfUUUHH-< HH6= Jrr query torch.Tensortensorstuple[torch.Tensor, ...] cum_seq_qtorch.Tensor | None require_fp32$tuple[tuple[str, torch.Tensor], ...]ctd|DsdStd|DdkrdS|jtjtjfvrdS|D]!\}}|jtjkr|dcS"||dkrd S||d krd Stj sd St|j d vrdSdS)Nc3$K|] }|jV dSr)is_cuda.0ts r z,_fa4_common_support_error.._s$**Qqy******rzinputs must be CUDA tensorsch|] }|j Sr )r!rTs r z,_fa4_common_support_error..as & & &AH & & &rr inputs must share devicez'query dtype must be float16 or bfloat16z dtype must be float32zdense query must be 4Dzragged query must be 3DzCUDA not available) z+FA4 requires compute capability 9.0 or 10.0) alllendtyper%float16bfloat16float32dimr& is_availabler*r!)rIrKrMrOnametensors r_fa4_common_support_errorriYs! **'*** * *-,, & &g & & &''1,,)) {5=%.99988$33 f <5= ( (222 2 2 2 )UYY[[A--''!1!1(( : " " $ $$##&&g55<< 4rkeyvalue dropout_pfloatreturn_debug_maskbool alibi_slopes seqused_k block_table num_splits int | Nonec |dkrdS|rdS|dS| |jtjkrdS|jsdSt |j} | | dkrd| d S| | d kr | dkrd | d St ||||f|} | | d krd S| SdS)Ndropout_p must be 0zreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDAr^z+paged KV (block_table) not supported on SM 0r z-SplitKV (num_splits > 1) not supported on SM rZz(query, key, value must be on same device)rar%int32rSr*r!ri) rIrjrkrlrnrprqrMrrrsr(errors r_fa4_forward_support_errorr{ssC$$100++ ?ek ) ),,  ,++ el + +E5B;;EUEEEE*q..Ub[[GuGGGG % U  E   . . .== 4rgrad_outout logsumexpc V|dkrdSt|||||||f|d|ff}||SdS)Nrvrwr~)rO)ri) r|rIrjrkr}r~rlrMrzs r_fa4_backward_support_errorrs\C$$ % 5#uc95"I.0    E   4rvalc|dkrdn|S)z"need to convert -1 to None for FA4Nr )rs r_aten_to_fa4_window_sizers"9944#%rTs Unpack[Ts]tuple[Unpack[Ts]]c4td|DS)Nc3BK|]}|ddVdS)r N) transposerTs rrWz#_transpose_dense..s044qQ""444444r)tuple)rKs r_transpose_densers 44G444 4 44rcu_seq_qcu_seq_kmax_qmax_kscale float | None is_causalwindow_size_leftwindow_size_right!tuple[torch.Tensor, torch.Tensor]c,ttdtt}||t| t| d||||| | nd| |pd| d }|j|||fi|\} }| |fS)NFA4 not registeredTr ) softmax_scalecausalrr return_lse cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_krq page_tablersr})rr7r/r contiguousr2)rIrjrkrrrrrrrrrqr}rrrsr8kwargslses r_fa4_run_forwardrs"/000  0 1 1F45EFF56GHH  /8/DY))+++$! oAF&v%eS%BB6BBHC    rF deterministic/tuple[torch.Tensor, torch.Tensor, torch.Tensor]c ttdtt} | |||||||| t | t | |||  \}}}|||fS)Nr)rrrrrrr)rr7r/r3rr)r|rIrjrkr}r~rrrrrrrr8dqdkdvs r_fa4_run_backwardrs/000  0 1 1F''    12BCC23DEE#(JBB r2:rT rrrrqrpr}rrcompute_auxiliaryrs cum_seq_krc t||||| || ||| }|td|t|||||||| || | | |||\}}|rntjdtj|j}tjdtj|j}tjd|j|j}nd}d}d}|||||fS)Nz)FA4 flash_attention forward unsupported: )r)rar!r r) r{r7rr%zerosuint64r!emptyra)rIrjrkrMrrrrlrrnrrrrqrpr}rrrrsrzr rng_state philox_offset debug_masks rrCrC s , '     E NuNNOOO       HC"KELNNN  Bel5<PPP [%+elKKK    Y z 99r)rrrrqrprrrsc Vt||||||||| | | | | ||||d|\}}}}}|S)NFr)rC)r}rIrjrkrMrrrrlrrnrrrrqrprrrsr)rs rrDrDJsb*8     )+! 'OAsAq!* Jr)rrrrunusedct||||||| |}|td|tj}t |||||||||| ||| \}}}|||fS)Nz*FA4 flash_attention backward unsupported: )rr7r%$are_deterministic_algorithms_enabledr)r|rIrjrkr}r~rMrrrrlrrrrrrrzrrrrs rrErEws( (      E OOOPPP>@@M"     JBB r2:rrvrct|||||ddd}|td|t|||\}} } tj|} | dd} |d} | d}t|| | dd| ||||||  \}}}}}|d}|d}| |dd|||||f S)NzFA4 SDPA forward unsupported: r r)rr})r{r7rr% empty_likersizerC)rIrjrkrlrrnrrzqkvout_bhsdout_bshd max_q_flash max_k_flashr)rrrrrrs rrFrFs. '        E CECCDDDuc511GAq! &&H!!!Q''H&&))K&&))K3T       4 4 40AsI}j JJqMME HHQKKE       r philox_seedrcTt||||||| d}|td|t|||||\}}}}}|d}|d} t ||||||dd|| | | | | |\}}}t|||\}}}|||fS)NzFA4 SDPA backward unsupported: rr)rr7rrrE)r|rIrjrkr}r~rMrrrrlrrrrrzrrrogorrrs rrGrGs$ (       E DUDDEEE%eS%hGGNAq!Q JJqMME HHQKKE3         JBB"""b"--JBB r2:rFA4) register_fn)r!r"rr#)r+)r,r-rr)r,r-rr )rr )r ) rIrJrKrLrMrNrOrPrr)NN)rIrJrjrJrkrJrlrmrnrorprNrqrNrMrNrrrNrsrtrr)r|rJrIrJrjrJrkrJr}rJr~rJrlrmrMrNrr)rrtrrt)rKrrr)NNN) rIrJrjrJrkrJrrNrrNrrtrrtrrrrorrtrrtrqrNr}rNrrrNrsrtrr)F)r|rJrIrJrjrJrkrJr}rJr~rJrrNrrNrrrrorrtrrtrrorr)&rIrJrjrJrkrJrMrNrrNrr#rr#rlrmrrornrorrrrtrrtrqrNrprNr}rNrrrNrrorsrt)$r}rJrIrJrjrJrkrJrMrNrrNrr#rr#rlrmrrornrorrrrtrrtrqrNrprNrrrNrsrt)"r|rJrIrJrjrJrkrJr}rJr~rJrMrNrrNrr#rr#rlrmrrorrJrrJrrrrtrrt)rvFF)rIrJrjrJrkrJrlrmrrornrorr)r|rJrIrJrjrJrkrJr}rJr~rJrMrNrrNrr#rr#rlrmrrorrJrrJrr)+__doc__ __future__rr4 dataclassesr functoolsrtypingrrtyping_extensionsrr r typesr r% torch.libraryr __all__rrrr*rr/r0rir{rrrrrrrCrDrErFrGregister_flash_attention_implr rrrsx#"""""!!!!!!%%%%%%%%22222222!      !!!!!!#  $####   3 / / / / /4:< F(,!%%%%%P.&&&& \$5555" $'+!%!%!%!%!%!j !!!!!`#'$(%)(,#'+"!)=:=:=:=:=:=:Z#'$(%)(,'+!'******z#'$(%000000n# :::::::Z!333333l( ';WXXXXXXr