
    <Цid             &       /   % S SK r S SKrS SKrS SK7  S SKJrJr  S SKrS SKJs  J	r
  S SKJr  S SKJr  SSKJr  / r\\   \S'   0 r\\\4   \S	'   SS
 jr   SS jrSS jrS\SS4S jrS\S\S\S\SS4
S jrS rS rS r\ R>                  " \\5      r S\\!   4S jr"S r#S r$S r%S r&\ " \RN                  RP                  RR                  RT                  \RN                  RP                  RV                  RT                  \RN                  RP                  RX                  RT                  \RN                  RP                  RZ                  RT                  \RN                  RP                  R\                  RT                  \RN                  RP                  R^                  RT                  \RN                  RP                  R`                  RT                  /S5      S 5       r1\ " \RN                  Rd                  Rf                  RT                  S5      S 5       r4\ " \RN                  RP                  Rj                  RT                  /S5      S  5       r6\ " \RN                  RP                  Rn                  RT                  S5      S! 5       r8\ " \RN                  RP                  Rn                  Rr                  S"5      " \85        \ " \RN                  RP                  Rt                  RT                  S#5      S$ 5       r;\ " \RN                  RP                  Rx                  RT                  S%5      S& 5       r=\ " \RN                  RP                  R|                  RT                  S'5      S( 5       r?\ " \RN                  RP                  R                  R                  S)5      S* 5       rB\ " \RN                  RP                  R                  RT                  S5      S+ 5       rD\ " \RN                  RP                  R                  RT                  S,5      S- 5       rF\ " \RN                  RP                  R                  RT                  S5      " \$5        \ " \RN                  RP                  R                  RT                  \RN                  RP                  R                  RT                  \RN                  RP                  R                  RT                  \RN                  RP                  R                  RT                  /S5      S. 5       rL\ " \RN                  RP                  R                  RT                  S5      S/ 5       rN\ " \RN                  RP                  R                  RT                  S05      S1 5       rP\ " \RN                  RP                  R                  RT                  S25      S3 5       rR\ " \RN                  RP                  R                  RT                  S45      S5 5       rT\ " \RN                  RP                  R                  RT                  S65      S7 5       rV\ " \RN                  RP                  R                  R                  S85      S9 5       rY\ " \RN                  RP                  R                  RT                  S:5      S; 5       rZ\ " \RN                  RP                  R                  R                  S<5      S= 5       r]\ " \RN                  RP                  R                  RT                  S>5      S? 5       r_\ " \RN                  RP                  R                  RT                  S@5      SA 5       r`\ " \RN                  RP                  R                  RT                  SB5      SC 5       rb\ " \RN                  RP                  R                  R                  SD5      SE 5       re\ " \RN                  RP                  R                  RX                  SF5      SG 5       rg\ " \RN                  RP                  R                  RT                  SH5      SI 5       ri\ " \RN                  RP                  R                  RT                  SJ5      SK 5       rk\ " \RN                  RP                  R                  RT                  SL5      SM 5       rm\ " \RN                  RP                  R                  RT                  SN5      SO 5       ro\ " \RN                  RP                  R                  RT                  SP5      SQ 5       rq\ " \RN                  RP                  R                  RT                  SR5      SS 5       rs\ " \RN                  RP                  R                  RT                  ST5      SU 5       rt\ " \RN                  RP                  R                  RT                  SV5      SW 5       ru\ " \RN                  RP                  R                  R                  SX5      SY 5       rx\ " \RN                  RP                  R                  RT                  SZ5      S[ 5       rz\ " \RN                  RP                  R                  RT                  SZ5      S\ 5       r|\ " \RN                  RP                  R                  RT                  S]5      S^ 5       r~S_ r\ " \RN                  RP                  GR                   RT                  S:5      S` 5       r\ " \RN                  RP                  GR                   GR                  Sa5      Sb 5       r\ " \RN                  RP                  GR                  R                  Sc5      Sd 5       r\ " \RN                  RP                  GR                  RT                  Se5      Sf 5       r\ " \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  /ST5      Sg 5       r\ " \RN                  RP                  GR                  RT                  Sh5      Si 5       r\ " \RN                  RP                  GR                  RT                  Sj5      Sk 5       r\ " \RN                  RP                  GR                  R                  Sl5      Sm 5       r\ " \RN                  RP                  GR"                  R                  Sn5      So 5       r\ " \RN                  RP                  GR&                  RT                  Sp5      \ " \RN                  RP                  GR(                  RT                  Sp5      Sq 5       5       r\ " \RN                  RP                  GR*                  RT                  Sr5      Ss 5       r\ " \RN                  RP                  GR.                  RX                  Sa5      St 5       r\ " \RN                  RP                  GR.                  RT                  S:5      Su 5       r\ " \RN                  RP                  GR4                  GR6                  Sv5      Sw 5       r\ " \RN                  RP                  GR4                  RX                  Sx5      Sy 5       r\ " \RN                  RP                  GR<                  GR6                  Sv5      Sz 5       r\ " \RN                  RP                  GR<                  RX                  Sx5      S{ 5       r\ " \RN                  RP                  GR<                  RT                  \RN                  RP                  GR4                  RT                  \RN                  RP                  GRB                  RT                  \RN                  RP                  GRD                  RT                  /S5      S| 5       r\ " \RN                  RP                  GRD                  RX                  Sx5      S} 5       r\ " \RN                  RP                  GRB                  RX                  Sx5      S~ 5       r\ " \RN                  RP                  GRL                  RT                  Sv5      S 5       r\ " \RN                  RP                  GRP                  RT                  Sv5      S 5       r\ " \RN                  RP                  GRT                  RT                  Sv5      S 5       r\ " \RN                  RP                  GRX                  RT                  Sv5      S 5       r\ " \RN                  RP                  GR\                  RT                  SJ5      S 5       r\ " \RN                  RP                  GR`                  RT                  S5      S 5       r\ " \RN                  RP                  GRd                  RT                  S5      S 5       r\ " \RN                  RP                  GRh                  RT                  \RN                  RP                  GRj                  RT                  /S5      S 5       r\ " \RN                  RP                  GR<                  RT                  S5      S 5       r\ " \RN                  RP                  GRp                  RT                  S5      S 5       r\ " \RN                  RP                  GRt                  RT                  S5      S 5       r\ " \RN                  RP                  GRx                  RT                  S5      S 5       r\ " \RN                  RP                  GR|                  RT                  S5      S 5       r\ " \RN                  RP                  GR~                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  /S5      S 5       r\ " \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  /S5      S 5       r\ " \RN                  RP                  GR                  GR                  S5      S 5       rS SKJrJr  S SKJr  \GR                  " \5        SS\R                  S\R                  S\R                  S\!S\S\S\\\4   S\S\S\\R                  \R                  4   4S jj5       r\GR                  " \5        SS\R                  S\R                  S\R                  S\R                  S\R                  S\R                  S\R                  S\\!\4   S\S\S\S\\\4   S\S\S\\R                  \R                  \R                  \\\R                     S4   4   4S jj5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\GR                  GR                  SS5       r(\(GR                  S\S5        \(GR                  S\S5        \(GR                  S\S5        SSS5        g! , (       d  f       g= f)    N)*)ListOptional)normalize_function)#jagged_scaled_dot_product_attention   NestedTensor__all__JAGGED_OPS_TABLEc                 $  ^ ^ SSK Jn  [        U[        [        45      (       a@  [        U5      " U U4S jU 5       5      n[        U5      " [        R                  U5      5      $ U(       a	  U" T U5      nUS:  a  UT :  d   eUS:X  a  TS-
  $ US-
  $ )Nr   canonicalize_dimsc              3   >   >#    U  H  n[        TUT5      v   M     g 7fN_outer_to_inner_dim).0dndim
ragged_dims     Y/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/torch/nested/_internal/ops.py	<genexpr>&_outer_to_inner_dim.<locals>.<genexpr>   s     QS.tQ
CCS   r   )torch._prims_commonr   
isinstancetuplelisttypedictfromkeys)r   dimr   canonicalizer   outputs   ` `   r   r   r      s    5#t}%%cQSQQF|DMM&122c*!8d
"" !AX:>2372    c                     SSK Jn  U" X5      nX:X  a  U(       d  [        U S35      eUS:X  a  U(       d  [        U S35      eU(       a  [        XU5      OUn	U(       a  US:H  n
X4$ U	$ )Nr   r   z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)r   r   RuntimeErrorr   )r   r#   r   op_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr   wrappedretoperating_on_batchs              r   _wrap_jagged_dimr0   '   s     6*G%5gY&VWXX	AogY&QRSS   	D:6 
  %\((Jr&   c                 @  ^ ^ SSK Jn  [        U[        [        45      (       d   S[        U5       35       eU Vs/ s H  oT" T U5      PM     nnSU;   nTU;   n[        U4S jU 5       5      n	[        [        R                  U U4S jU 5       5      5      n
XX4$ s  snf )z~
For NestedTensor operators,
wraps dimensions to non-negative values,
and returns metadata related to reduction dimension(s).
r   r   z<_wrap_jagged_dims(): cannot iterate over dimensions of type c              3   D   >#    U  H  oS :g  =(       a    UT:g  v   M     g7f)r   N )r   r   
ragged_idxs     r   r   $_wrap_jagged_dims.<locals>.<genexpr>V   s     P<aAv9!z/9<s    c              3   >   >#    U  H  n[        TUT5      v   M     g 7fr   r   )r   r   r   r4   s     r   r   r5   Z   s     U1)$:>>r   )	r   r   r   r   r   r    anyr!   r"   )r   dimsr)   r4   r   r   wrapped_dimsoperate_on_batchoperate_on_raggedoperate_on_non_batchouter_to_inner_dims   `  `       r   _wrap_jagged_dimsr>   D   s     6udm  S	Ed4j\RS 
 -1,0q$"D   L("l2P<PP UUU 1BXXs   B
schema_strreturnc                 h  ^ U R                  S5      nU Vs/ s H  oUR                  S5      PM     snR                  S5      n[        U5      U-
  nUS   S:X  a  US S nO^[        U5      U:  a  [        U5      [        U5      ::  d7  [	        SUR
                   SU  SU S	[        U5       S
[        U5       S35      eS S S S S.n[        U5       H  u  pU
R                  S5      u  pUR                  S5      nU(       a  US S OUnXR                  5       ;  a  [        SU 35      eU	[        U5      :  a'  U(       d  [	        SUR
                   SU  SU 35      eM  X   mU4U4S jjnU" X)   5      (       a  M  SSSSSS.n[	        SUR
                   SU  SU SUU    35      e   g s  snf )N, ?T...NestedTensor (z): expected at least z arguments and at most z arguments, but got: z
 argumentsc                 n    [        U [        R                  5      =(       a    [        U [        5      (       + $ r   )r   torchTensorr
   xs    r   <lambda>check_schema.<locals>.<lambda>q   s!    z!U\\2V:a;V7VVr&   c                 x    [        U [        5      =(       a$    U R                  S L =(       a    U R                  S:H  $ Nr   )r   r
   _lengths_ragged_idxrK   s    r   rM   rN   r   s2    
1l3 JJ$MMQr&   c                 "    [        U [        5      $ r   r   r
   rK   s    r   rM   rN   u   s    J|
r&   c                     g)NTr3   rK   s    r   rM   rN   x   s    r&   )tjtjt_allr7   z: zUnknown arg type: z) missing required argument: c                 H   > U(       a  U S L =(       d    T" U 5      $ T" U 5      $ r   r3   )rL   is_optional	_check_fns     r   check_fncheck_schema.<locals>.check_fn   s#    Dy0IaL0 |#r&   tensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz
<any type>)rV   zt?rW   rX   r7   z): expected z	 to be a )	splitendswithcountlen
ValueError__name__	enumeratekeysAssertionError)r?   funcargskwargsnamed_arg_typesrL   num_optional_argsmin_argsarg_type_check_fnsinamed_arg_typenamearg_typerZ   normalized_arg_typer\   type_to_descr[   s                    @r   check_schemaru   `   s    &&t,O2AB/QC/BHHN?#&77H re#)#2.D	X%#d)s?7K*Ka
|;PQYPZ [)),_)=(>>St9+Z)  W
 	 '7'--d3'',/:hsm&=&=&?? #56I5J!KLLD	> #DMM?!J< @2269  &;	$/ 	$   '=6#L a
|<vY)*, A 81 Cs   F/aa_namebb_namec           	          UR                   UR                     UR                   UR                     :w  a  [        SU R                   SU SU S35      eg )NrF   z: expected  and z' to have the same exact offsets tensor.)_sizerR   r(   rd   )rh   rv   rw   rx   ry   s        r   check_ragged_dim_samer}      sZ     	wwq}}!77DMM?+fXU6( K) )
 	
 8r&   c                     U R                   S-   nU R                  S U nUS U n[        U5      [        U5      :H  =(       a    [        S [	        X45       5       5      $ )Nr   c              3   F   #    U  H  u  pX:H  =(       d    US :H  v   M     g7f)rD   Nr3   )r   nsss      r   r   %raggedness_matches.<locals>.<genexpr>   s"     H,G52BGqBw,Gs   !)rR   r|   rb   allzip)ntsizeend	nt_raggedsize_raggeds        r   raggedness_matchesr      sX    
..1
C#It*Ky>S-- HC	,GHHr&   c                     U R                  5       S:  aM  U R                  S   S:X  a:  U R                  S5      n U R                  5       S:  a  U R                  S   S:X  a  M:  U $ Nr   r   )r#   shapesqueeze)rV   s    r   squeeze_leading_onesr      sN    & %%'A+!''!*/IIaL %%'A+!''!*/Hr&   c                 |   ^ ^^ [        T[        5      (       d  T/m[        T [        5      (       d  T /m UUU 4S jnU$ )Nc                 P   >^  T H  nU U4S jnT H  nU" U5      X1'   M     M     T $ )Nc                    >^  U UU4S jnU$ )Nc                  <   > [        TT/U Q70 UD6  T" T/U Q70 UD6$ r   )ru   )ri   rj   aten_oprh   r?   s     r   inner@register_func.<locals>.wrapper.<locals>.get_inner.<locals>.inner   s,     TCDCFC9$9&99r&   r3   )r   r   rh   r?   s   ` r   	get_inner1register_func.<locals>.wrapper.<locals>.get_inner   s    : r&   r3   )rh   r   r   tableaten_opsr?   tabless   `   r   wrapperregister_func.<locals>.wrapper   s1    G  !*7!3     r&   )r   r   )r   r   r?   r   s   ``` r   register_funcr      s7    h%%:fd## Nr&   c                 0   [         R                  U S 5      nUb  U$ [        R                  R                  U R
                  ;   GaR  SSKJn  U H+  nU" U5      (       d  M  [        SU R                   SU 35      e   [        S U 5       5      nUS:X  a  / nU R                  R                   Hj  n[        UR                  [        R                  5      (       a   UR!                  UR"                   S35          O"UR!                  UR"                   S35        Ml     UR!                  S	5        S
R%                  U5      n['        X/UQ70 UD6  [(        R*                  " [,        U 5      $ US:X  a+  ['        SU /UQ70 UD6  [(        R*                  " [.        U 5      $ g )Nr   )is_nested_intrF   z: invalid argument c              3   V   #    U  H  n[        U[        R                  5      v   M!     g 7fr   )r   rI   rJ   )r   rL   s     r   r    lookup_jagged.<locals>.<genexpr>   s     H4ajELL994s   ')r   z: jt_allz: anyrE   rB      zlhs: any, rhs: any, ...)r   getrI   Tag	pointwisetags%torch.fx.experimental.symbolic_shapesr   r(   rd   sum_schema	argumentsr   r    
TensorTypeappendrq   joinru   	functoolspartialjagged_unary_pointwisejagged_binary_pointwise)	rh   ri   rj   dispatch_funcr   argnum_tensor_argsschema_partscheck_schema_strs	            r   lookup_jaggedr      so   $((t4M  yydii'G
 CS!!"#DMM?2EcUK   H4HHa L||--chh(8(899 ''388*H(=> ''388*E(:; . &#yy6)A$A&A$$%;TBB!2DJ4J6J$$%<dCCr&   c                 t    U R                  5       U R                  5       U R                  U R                  S.nU$ )Noffsetslengths_metadata_cacherR   r   )r   rj   s     r   extract_kwargsr     s2    ;;=;;=..	F Mr&   c                 r   ^ [        S U 5       5      m[        U " U4S jU 5       0 UD640 [        T5      D6$ )Nc              3   T   #    U  H  n[        U[        5      (       d  M  Uv   M      g 7fr   rT   r   r   s     r   r   )jagged_unary_pointwise.<locals>.<genexpr>  s     Ddsjl&Cssd   (	(c              3   H   >#    U  H  oTL a  UR                   OUv   M     g 7fr   )_values)r   r   njts     r   r   r     s     BTcSjs{{c1Ts   "nextr
   r   )rh   ri   rj   r   s      @r   r   r     s@    
DdD
DCBTBMfM

 r&   c           	         US   US   pC[        U[        5      (       d  [        U[        5      (       d   eSn[        U[        5      (       a  [        U[        5      (       a  [        X4R                  5      (       a7  [        U " UR                  UR                  /USS  Q70 UD640 [        U5      D6$ [        UR                  U R                  UR                  UR                  5      5      e[        U[        5      nU(       a  [        U5      O
[        U5      nU(       a  X44OXC4u  pU	R                  5       UR                  5       :  a  [        S5      e[        U	5      n
UR                  5       U
R                  5       S-   :  a=  U(       a  UR                  U
4OXR                  4u  p[        U " X/USS  Q70 UD640 UD6$ UR                  5       UR                  5       :X  Ga0  UR                  S   UR                  S   :w  a:  [        UR                  U R                  UR                  UR                  5      5      eSSKJn  UR                  nUR                   nUnUR                  R                  UR"                  S-
     nUc  UnU	R%                  ['        U	R                  5       VVs/ s H  u  nnUUR"                  :w  a  UOUPM     snn5      n	U" U	UR(                  UR"                  UUUS9nU(       a  UU4OUU4u  pU " X/USS  Q70 UD6$ [        UR                  U R                  UR                  UR                  5      5      es  snnf )Nr   r   zHcannot call binary pointwise function {} with inputs of shapes {} and {}r   z+NYI: broadcasting NT with T with larger dimnested_from_paddedr   r4   sum_S
min_seqlen
max_seqlen)r   r
   r   r|   r   r   r(   formatrd   r#   NotImplementedErrorr   r   nested_tensorr   _maybe_min_seqlen_maybe_max_seqlenrR   expandre   _offsets)rh   ri   rj   rv   rx   mismatch_error_msga_is_ntextracted_kwargsr   rV   
t_squeezedlhsrhsr   r   r   padded_max_Stotal_Lro   rL   t_as_nts                        r   r   r   $  s   7DGqa&&*Q*E*EEE 	S  !\""z!\'B'B a))QYY		?DH??CQRSCT  -44T]]AGGQWWUVVL)G,3~a(9J QFA6EBuuw!"OPP%a(J	vvx:>>#a''/6BJJ
+Z<TD?DH??TCSTT 	uuw!%%' 771:#"))$--!''J  	6 ))
))
!**""2>>A#56"L HHDMaggDVWDVDAq!r~~%Q<7DVW
 %KK~~!!
 %,B='2C2tABx2622 )00Q
RR# Xs   .!M
c                 B   U [         R                  R                  R                  L a  [	        U0 UD6$ U R
                  S:X  a!  U " US   R                  /USS  Q70 UD6  US   $ U R
                  S:X  a  SS jn[        X1USS9u  pEUR                  S5      n[        UR                  5       US	   UR                  SS
S9n[        UR                  5       US   UR                  SS
S9nXx:X  a  U$ [        R                  " [        R                  UR                   XxS-    5      n	/ UR                   S U QU	PUR                   US-   S  Q7n
UR"                  " U
6 $ U R
                  S:X  a  SS jn[        XUSS9u  pEUR                  S5      nUR                  S5      nUR                  5       UR                  -
  S-
  n[%        U5      U:  a  ['        S5      e[         R                  R)                  5          U " U0 UD6sS S S 5        $ [+        U 5      e! , (       d  f       N= f)Napply_r   r   flattenc                     g r   r3   )input	start_dimend_dims      r   _flatten_sig+jagged_torch_function.<locals>._flatten_sig      r&   Tri   rj   normalize_to_only_use_kwargsr   r   F)r*   r   rms_normc                     g r   r3   )r   normalized_shapeweightepss       r   _rms_norm_sig,jagged_torch_function.<locals>._rms_norm_sig  r   r&   r   zNrms_norm(): Normalization over the ragged dim not supported for nested tensors)r   rD   )NN)rI   _C_nnscaled_dot_product_attentionr   rd   r   r   popr0   r#   rR   r   reduceoperatormulr   reshaperb   rc   DisableTorchFunctionSubclassr   )rh   ri   rj   r   _
new_kwargsinpr   r   product	new_shaper   r   max_normalizables                 r   jagged_torch_functionr  r  s    uxx||8882DCFCC}} T!W__2tABx262Aw }}	!	 +FQU
 nnW% %GGI{#OO!&
	 #GGIy!OO!&
 J""8<<9QR{1STPcii
+PWPsyy17OP	{{I&& }}
"	 +VRV
 nnW%%>>*<= 779s6: #33`  XX224(( 54 d
## 54s   3H
Hzself: jt_allc                 J   U [         R                  R                  R                  R                  :X  a  gU [         R                  R                  R
                  R                  :X  a  US   R                  $ U [         R                  R                  R                  R                  :X  a  [        US   R                  5      $ U [         R                  R                  R                  R                  [         R                  R                  R                  R                  4;   au  US   R                  bH  [        [        US   R                  5      [        R                  " US   R                  SS  5      -  5      $ US   R                   R                  5       $ U [         R                  R                  R"                  R                  :X  a  US   R$                  $ U [         R                  R                  R&                  R                  :X  a  US   R                   R)                  5       $ g )NFr   r   )rI   opsatenis_non_overlapping_and_densedefaultsym_sizer|   r#   rb   	sym_numelnumelrQ   intr   mathprodr   
sym_stride_stridessym_storage_offsetstorage_offsetrh   ri   rj   s      r   tensor_attr_supported_getterr    ss    uyy~~::BBBuyy~~&&...Aw}}uyy~~!!)))47==!!		((00%))..2F2F2N2NOO7's47++,tyyaqr9J/KKLLAw$$&&uyy~~((000Awuyy~~00888Aw--// 9r&   c                 "    [         R                  $ r   )rI   jaggedr  s      r   prim_layout_defaultr    s    <<r&   c                 ~    U [         R                  R                  R                  R                  :X  a  [        S5      eg )NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)rI   r  r  r   r  r(   r  s      r   tensor_attr_unsupported_getterr    s7    
 uyy~~""***9
 	
 +r&   c                 
   SSK Jn  [        XUSS9u  pEUR                  S5      nUR	                  5       b  gUR                  S[        R                  5      US'   US   [        R                  :X  a  gU" UR                  40 UD6$ )Nr   )is_contiguous_for_memory_formatTr   r   Fmemory_format)
r   r  r   r   r   r   rI   contiguous_formatpreserve_formatr   )rh   ri   rj   r  r   r   r   s          r   is_contiguous_generalr    s    C&TMA ..
!C {{} ",..00#J /"e&;&;;*3;;E*EEr&   z!self: jt_all, memory_format: any?z"input: jt_all, memory_format: any?c                 B   [        XUSS9u  p4UR                  S5      n[        U5      nUR                  bM  US   [        R
                  :X  a6  SSKJn  UR                  S:X  d   S5       eU" UR                  5       S S9u  pU$ [        U " UR                  40 UD640 UD6$ )	NTr   r   r  r   )jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r   )r   r   r   rQ   rI   r  r   r   rR   unbindr
   r   )	rh   ri   rj   r   r   r   new_metar   contigs	            r   clone_defaultr$    s     'TMA ..
!Cc"H
||o&%*A*AA 8 1$MLM$(tDIFMS[[7J7D8DDr&   zinput: jt, weight: t, bias: t?c                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ NTr   r   r   r   r
   r   r   rh   ri   rj   r   r   r   s         r   linear_defaultr)  #  H    &TMA ..
!CS[[7J7O>#;NOOr&   z6self: jt, grad_output: jt, weight: t, output_mask: anyc                    [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      nSu  pn[        XSUS5        US	   (       a4  [        [        R
                  " UR                  U5      40 [        U5      D6n	US
   (       a{  UR                  R                  SUR                  S	5      5      nUR                  R                  SUR                  S
5      5      n[        R
                  " UR                  5       U5      n
US   (       a  UR                  nXU4$ )NTr   r   grad_outputr   output_mask)NNNselfr   r   rD   r   )r   r   r}   r
   rI   matmulr   r   r   r   rV   )rh   ri   rj   r   r   r   r,  r   r-  dsdwdbgrad_2dinput_2ds                 r   linear_backward_defaultr5  .  s   
 'TMA ..
!C../K^^H%F../K!JBB$V[-H1~LL,,f5
9G9T
 1~ %%--b&++a.A;;&&r6;;q>:\\'))+x01~  B<r&   zinput: jt_all, dtype: anyc                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ r&  r'  r(  s         r   to_dtyper7  Q  r*  r&   c                 j   SSK Jn  [        XUSS9u  pEUR                  S5      nUR                  S5        U " UR                  40 UD6nUR
                  R                  UR                  S9nS n	UR                  b#  UR                  R                  UR                  S9n	SS	K	J
n
  SS
KJnJn  UR                  c  UR
                  OUR                  nU	c  UOU	n[        XU45      (       a"  U" U5      nU" U5      nUR                  Ul        OX=   X>'   [!        U5      nUUS'   U	US'   [#        U40 UD6nU$ )Nr   )_tensor_symint_registryTr   r   layout)devicer   )
FakeTensor)FunctionalTensormb_unwrap_functional_tensorr   r   )r   r9  r   r   r   r   tor;  rQ   torch._subclasses.fake_tensorr<  #torch._subclasses.functional_tensorr=  r>  r   nested_int_memor   r
   )rh   ri   rj   r9  r   r   r   
new_valuesnew_offsetsnew_lengthsr<  r=  r>  ragged_source	new_thingtgtsrc
inp_kwargsr%   s                      r   to_copy_defaultrK  \  s)   6&TMA ..
!CNN8ckk0Z0J,,//):):/;KK
||llooZ->->o?8
 %(LL$8CLLcllM*2I)*:;<<))4)-8!11-D-S*$J'Jy'Jy*3
3FMr&   z-self: jt_all, src: jt_all, non_blocking: any?c                     [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  UR                  :w  a  [        S5      eUR                  R                  UR                  5        U$ )NTr   r   rI  zXcopy_ only supports Nested Tensors that have same size and the exact same offset tensor.)r   r   r|   r(   r   copy_)rh   ri   rj   r   r   r   rI  s          r   copy_defaultrN    sr     'TMA ..
!C
..
C
yyCIIf
 	
 KKckk"Jr&   c                     [        XUSS9u  p4UR                  S5      n[        R                  US'   [	        U " UR
                  40 UD640 [        U5      D6$ )NTr   r   r:  )r   r   rI   stridedr
   r   r   r(  s         r   like_factory_defaultrQ    sZ     'TMA ..
!C
 !==JxS[[7J7O>#;NOOr&   c                 f    [        XUSS9u  p4UR                  S5      nU " UR                  5        U$ r&  r   r   r   r(  s         r   zero__defaultrT    s6    &TMA ..
!CJr&   z*self: jt_all, dim: any, half_to_float: anyc           
      d   [        XUSS9u  p4[        US   [        5      (       a  [        S5      eUR	                  S5      n[        UR                  5       US   4SUR                  5      u  US'   nnnU(       a  [        S5      eU(       a  UR                  S:  a  [        S	5      eU(       a  UR                  b  [        S
5      eUS   S   US'   U(       Ga2  [        R                  R                  R                  [        R                  R                  R                  UR                   R#                  UR                   R$                  S   S5      UR&                  /UR(                  /[+        S5      S9UR                  S9n	[        R                  R                  R-                  U	UR&                  /UR                   R$                  S   S9R"                  " S/UR                   R$                  SS  Q76 n
[/        U
40 [1        U5      D6$ [/        U " UR                   40 UD640 [1        U5      D6$ )NTr   r#   zHsoftmax(): not supported for dimensions of type 'tuple' for NestedTensorr   softmaxzRsoftmax(): not supported when reducing across the batch dimension for NestedTensorr   zesoftmax(): not supported when reducing along the ragged dimension for ragged_idx > 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorr   rD   z-inf)max_lengthspadding_valuer#   r   )r   r   r   r(   r   r>   r#   rR   rQ   rI   nn
functionalrV  r  r  _jagged_to_padded_dense_forwardr   r   r   r   _max_seqlenfloat_padded_dense_to_jagged_forwardr
   r   )rh   ri   rj   r   r   r   reduce_on_batchreduce_on_ragged_reduce_on_non_batchpadded_softmax_valuessoftmax_valuess              r   _softmax_defaultrf    s/    'TMA *U#U++V
 	
 ..
!C 			E			5 `
 	
 COOa/s
 	
 CLL4I
 	

 #5)	Ju  % 3 3 ; ;IINN::##KK%%a("  __-#Fm ;   !< 
!
 GG!\\NKK%% H 
 ' 
 ""12&
 NBnS.ABBS[[7J7O>#;NOOr&   z7grad_output: jt, output: jt, dim: any, input_dtype: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " UR                  UR                  40 UD640 [	        U5      D6$ )NTr   r,  r%   r'  )rh   ri   rj   r   r   grad_outr%   s          r   _softmax_backwardri    sh    
 'TMA ~~m,H^^H%FXv~~<<@Nx@X r&   z!self: jt, float: any, train: any?c                     [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6u  pg[        U40 [	        U5      D6[        U40 [	        U5      D64$ r&  r   r   r   r
   r   )rh   ri   rj   r   r   r   out1out2s           r   native_dropout_defaultrn    sj     'TMA ..
!Cckk0Z0JDT1^C01T1^C01 r&   z%grad_output: jt, mask: jt, scale: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " UR                  UR                  40 UD640 [	        U5      D6$ )NTr   r,  maskr'  )rh   ri   rj   r   r   r,  rp  s          r   native_dropout_backward_defaultrq  ,  sh    
 'TMA ../K>>&!D[  $,,=*=

% r&   z2self: jt_all, dim: any, keepdim: any?, dtype: any?c                 $    [        U SS/UQ70 UD6$ )Nr  r   _apply_reductionr  s      r   prod_dim_intru  <  s    
 D&!=d=f==r&   zself: jt_all, dtype: any?c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r&  rS  r(  s         r   prod_defaultrw  D  9    &TMA ..
!C*z**r&   z$self: jt, split_size: any, dim: any?c                    ^ [        XUSS9u  p4UR                  S5      m[        TR                  5       US   TR                  S5      US'   [        U4S jU " TR                  40 UD6 5       5      $ )NTr   r   r#   r_   c              3   P   >#    U  H  n[        SS U0[        T5      D6v   M     g7f)valuesNr3   )r
   r   )r   rL   r   s     r   r   split_tensor.<locals>.<genexpr>]  s*      0A 	5A5!450s   #&)r   r   r0   r#   rR   r   r   r(  s        @r   split_tensorr}  O  sy     'TMA ..
!C(	:e$coowJu  ckk0Z0  r&   z%self: jt, split_sizes: any, dim: any?c           
         [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  S5      US'   U " UR
                  40 UD6 Vs/ s H  n[        SSU0[        U5      D6PM     sn$ s  snf )NTr   r   r#   split_with_sizesr{  r3   r   r   r0   r#   rR   r   r
   r   )rh   ri   rj   r   r   r   rL   s          r   split_with_sizes_defaultr  c  s     'TMA ..
!C(	:e$coo7IJu ckk0Z00A 	5A5!450  s     Bz+self: jt, dim: any, start: any, length: anyc                     [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  S5      nU " UR
                  UUS   US   S9n[        U40 [        U5      D6$ )	NTr   r   r#   narrowstartlengthr#   r  r  r  )rh   ri   rj   r   r   r   r#   r{  s           r   r  r  w  s     'TMA ..
!C
3779j&7(
SC!(#	F 6."566r&   z self: jt, chunks: any, dim: any?c           
         [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  SSS9u  US'   nU(       Ga?  US   nUR
                  S   n[        R                  " X-  5      n	UR                  R                  5       n
U
R                  U5      nU Vs/ s H  n[        R                  " USS	9PM     nnU Vs/ s H  n[        R                  " US
SS9PM     nnU Vs/ s H  nXR                  S.PM     nnU Vs/ s H   oR                  5       R!                  5       PM"     nnUR"                  R%                  U5      n['        S[)        U5      5       Vs/ s H  n[+        SSUU   0UU   D6PM     sn$ U " UR"                  40 UD6 Vs/ s H  n[+        SSU0[-        U5      D6PM     sn$ s  snf s  snf s  snf s  snf s  snf s  snf )NTr   r   r#   chunkr,   chunksr   rY  r   r   )value)r   rR   r{  r3   )r   r   r0   r#   rR   r|   r  ceilr   diffr  rI   cumsumFpadr   itemr   r_   rangerb   r
   r   )rh   ri   rj   r   r   r   r/   r  	dim0_size
chunk_sizer   chunked_lengthsrL   chunked_offsetsper_offsetsnested_kwargssplit_sizeschunk_valuesro   s                      r   chunk_defaultr    s   &TMA ..
!C,<	:e$coowPT-)Ju) H%IIaL	YYy12
 ,,##%!--/;JK?a5<<q1?K>MNo155F!4oN  /
. $OOD. 	 
 0??!uuw||~?{{((5 1c,/0
0 FQFM!4DF0
 	
 #++44
4 99^C%894
 	
% LN
 @



s$   0G G;G'G4G ) G%zself: jt_all, dim: any?c           
        ^^ [        XUSS9u  p4US   nUS:w  a  [        S5      eUR                  S5      nUR                  5       mUR	                  5       nUR                  5       nUR                  mSS[        [           S[        [        [              4UU4S	 jjjn	Uc?  UR                  5       R                  5       n
U	" U
5        [        R                  " TU
TS
-
  S9$ TS::  a  [        S5      eUR                  5       n
UR                  5       nU	" X5        [        UR                  S   5       Vs/ s H!  n[        R                   " TTS
-
  X   X   S9PM#     sn$ s  snf )NTr   r#   r   z2unbind(): only supported for NestedTensor on dim=0r   rQ   r   c                   > SnTR                   TS-
     n[        [        U 5      5       Hd  n[        R                  " X   5        [        R
                  " X   U:*  5        X U   -  nUc  MB  [        R
                  " X   X   -   U:*  S 5        Mf     [        R
                  " X#:*  5        UbO  [        [        U5      5       H6  n[        R                  " X   5        [        R
                  " X   U:*  5        M8     g g )Nr   r   c                      g)NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionr3   r3   r&   r   rM   2unbind_int.<locals>._torch_check.<locals>.<lambda>  s    kr&   )r   r  rb   rI   _check_is_size_check)rQ   r   lengths_sumragged_dim_sizero   r4   r{  s        r   _torch_check unbind_int.<locals>._torch_check  s      ,,zA~6s8}%A  -LL78A;&K#K(+-@k & 	[343x=)$$X[1X[O;< *  r&   r   rY  zAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)r  r   )r   r(   r   r{  r   r   rR   r   r  r   r  tolistrI   r_   r  r   r  )rh   ri   rj   r   r   r#   r   r   r   r  lengths_scalarsoffsets_scalarsro   r4   r{  s                @@r   
unbind_intr    s_    'TMA U
C
axOPP
..
!CZZ\FkkmGkkmGJ=tCy =HT#Y4G = =: !,,.//1_%{{6?aIIQO
 	
 nn&Onn&O2 w}}Q'( )A 	a!$"%		
 )  s   4(Ezself: jt, dim: anyc                     [        XUSS9u  p4UR                  S5      nUR                  n[        [	        UR
                  5      US   UR                  S5      US'   [        U " U40 UD640 [        U5      D6$ )NTr   r   r#   r   )	r   r   r   r0   rb   r|   rR   r
   r   )rh   ri   rj   r   r   r   r{  s          r   squeeze_dimr    sx    &TMA ..
!C[[F(CII
5)3??IJu V2z2JnS6IJJr&   zself: jt_all, dim: anyc                 @   [        XUSS9u  p4UR                  S5      nUR                  nUS   n[        [	        UR
                  5      S-   XuR                  SSS9US'   [        U5      nUS   UR                  S-
  ::  a  US==   S-  ss'   [        U " U40 UD640 UD6$ )	NTr   r   r#   r   	unsqueeze)r+   rR   )	r   r   r   r0   rb   r|   rR   r   r
   )	rh   ri   rj   r   r   r   r{  r#   output_kwargss	            r   unsqueeze_defaultr    s    &TMA ..
!C[[F U
C(CIIC+PTJu
 #3'M%COOa//m$)$V2z2DmDDr&   ztensors: any, dim: anyc                    [        XUSS9u  p4UR                  S5      nU Vs/ s H  ofR                  (       d  M  UPM     nn[        U5      S:  d   eUS   nU Vs/ s H&  ofR                  (       a  UOUR	                  U5      PM(     nnUS   n	[        [        UR                  5      XR                  S5      US'   [        U " U Vs/ s H  ofR                  PM     sn40 UD640 [        US   5      D6$ s  snf s  snf s  snf )NTr   tensorsr   r#   cat)r   r   	is_nestedrb   	expand_asr0   r   rR   r
   r   r   )
rh   ri   rj   r   r   r  rV   nestedfirstr#   s
             r   cat_defaultr  %  s    &TMA nnY'G !0AKKaF0v;??1IEAHIAKKqQ[[%77GI U
C(EKK#00%Ju )Aii)8Z8<J7ST:<V  1 J *s   C7C7-C<	Dzself: jt_all, other: anyc                   ^  [        T XSS9u  p4UR                  S5      nUR                  S5      nU 4S jnU 4S jnUR                  (       a  UR                  (       d  UR                  5       S:  a*  UR                  5       UR                  5       :X  a  U" XV5      $ UR                  5       S:X  aI  UR                  5       UR                  5       :  a'  [	        T " UR
                  U40 UD640 [        U5      D6$ GO2UR                  (       Ga   UR                  (       Ga  UR                  5       S:  a^  UR                  5       S:  aJ  [        XVR                  5      (       a0  [	        T " UR
                  UR
                  5      40 [        U5      D6$ UR                  5       S:X  a  UR                  5       S:X  at  UR                  S:X  ad  UR                  S	:X  aT  UR                  UR                  5      UR                  UR                  5      :X  a  [        R                  " U" XV5      5      $ [        S
UR                   SUR                   35      e)NTr   r   otherc                    > [        U R                  5       UR                  5       5       VVs/ s H  u  p#T" X#5      PM     snn$ s  snnf r   )r   r!  )rv   rx   a_compb_comprh   s       r   _unbind_impl$matmul_default.<locals>._unbind_implG  s@    9<QXXZ9T
9T%5fD 9T
 	
 
s   Ac           	        > U R                   (       a  UR                   (       a   eU nSSKJn  UR                  nUR                  nUnUR
                  R                  UR                  S-
     nUc  Un/ UR                  S UR                   QUPUR                  UR                  S-   S  Q7nUR                  SUS9n	U" T
" X5      UR                  UR                  UUUS9$ )Nr   r           output_sizer   )
r  r   r   r   r   r   r   rR   to_padded_tensorr   )rv   rx   r   r   r   r   r   r   padded_shape	padded_ntrh   s             r   _padded_impl$matmul_default.<locals>._padded_implL  s    {{1;;..5))
))
!**""2>>A#56"L
XX&'

 XXbnnq(*+

 '''F	!KK~~!!
 	
r&      r   r   z1matmul(): not supported between inputs of shapes r{   )r   r   r  r#   r
   r   r   r   r|   rR   r   rI   stackr(   r   )	rh   ri   rj   r   r   r   r  r  r  s	   `        r   matmul_defaultr  >  s   &4TMA ..
!CNN7#E


> }}U__779>cggi599;6++
 YY[A#'')eiik"9S[[%6:6:H:M  
5??? 779q=UYY[1_1CC1U1US[[%-- @XNSVDWXX GGIN		q 1$!!Q&)UZZ8I8I-JJ ;;|C788

;CII;eEKK=Y r&   zself: jt_all, mat2: anyc                 L   [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  5       S:w  a  [        S5      eUR                  5       S:w  a  [        S5      e[	        [
        R                  R                  R                  R                  XV5      $ )NTr   r   mat2r  zbmm(): input must be 3Dzbmm(): mat2 must be 3D)
r   r   r#   rc   r  rI   r  r  r/  r  rh   ri   rj   r   r   r   r  s          r   bmm_defaultr    s    &TMA ..
!CNN6"E
wwyA~233yy{a122%))..//77DDr&   z'self: jt_all, size: any, implicit: any?c                    [        XUSS9u  p4UR                  S5      nUS   nSU;  d  UR                  S5      (       a   e[        XV5      (       d  [        SUR                   SU 35      e[        SUR                  5       5       Vs/ s H  owUR                  :X  a  S	OXg   PM     nn[        U " UR                  U5      40 [        U5      D6$ s  snf )
NTr   r   r   implicitzexpand(): cannot expand shape z -> r   rD   )r   r   r   r(   r|   r  r#   rR   r
   r   r   )	rh   ri   rj   r   r   r   r   r   
expand_args	            r   expand_defaultr    s     'TMA ..
!CfDj(*..2L2LMMc((;CII;d4&QRRCHCGGICVWCVaS__,"$'9CVJWS[[*5M9LMM Xs   	Czself: t, other: jtc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " XVR                  5      40 [	        U5      D6$ )NTr   r   r  r'  r  s          r   expand_as_defaultr    sQ    &TMA ..
!CNN7#ES--0JN54IJJr&   zself: jt_all, size: anyc           
      .   [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U5      UR                  5       ::  a?  UR	                  / S [        UR                  5       [        U5      -
  5       5       QUQ5      $ [        S5      e)NTr   r   r   c              3   &   #    U  H  nS v   M	     g7f)r   Nr3   )r   r   s     r   r   broadcast_to.<locals>.<genexpr>  s     E(D1Q(Ds   zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)r   r   rb   r#   r   r  rc   )rh   ri   rj   r   r   r   r   s          r   broadcast_tor    s    &TMA ..
!C>>&!D
4yCGGIzzMEcggi#d).C(DEMMNN
	4 r&   ztensors: anyc           
      b   [        XUSS9u  p4UR                  S5      n[        U5      S:X  a  [        S5      e[        U5      S:X  a  US   $ / n[        R
                  " S U 5       6 n[        S U 5       5      nU H  n	U	R                  (       a"  UR                  U	R                  U5      5        M6  U	R                  5       [        U5      :  aI  UR                  [        U	R                  UR                  R                  5      40 [        U5      D65        M  [        S	5      e   [        U5      $ )
NTr   r  r   z7broadcast_tensors(): expected at least one tensor inputr   c              3   8   #    U  H  oR                   v   M     g 7fr   )r   r   rV   s     r   r   $broadcast_tensors.<locals>.<genexpr>  s     .H1wws   c              3   T   #    U  H  n[        U[        5      (       d  M  Uv   M      g 7fr   rT   r  s     r   r   r    s     A'QZ<%@qq'r   zubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)r   r   rb   rc   rI   broadcast_shapesr   r  r   r  r#   r
   r   r   r   r   )
rh   ri   rj   r   r   r  outsbroadcast_shaper   rV   s
             r   broadcast_tensorsr    s	   &TMA nnY'G
7|qRSS
7|qqzD,,.H.HIO
A'A
AC;;KK78UUWs?++KKQ^^CKK,=,=>V.QTBUV ;   ;r&   z(condition: jt_all, self: any, other: anyc                 .   [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  S5      n[        R                  " XVU5      u  pVn[	        U " UR
                  UR
                  UR
                  40 UD640 [        U5      D6$ )NTr   	conditionr   r  )r   r   rI   r  r
   r   r   )rh   ri   rj   r   r   r  r   r  s           r   
where_selfr    s     'TMA {+I
..
!CNN7#E "33IEJIEYU]]IjI

# r&   zself: jt, device: any?c                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ r&  r'  r(  s         r   _pin_memory_defaultr    r*  r&   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r&  rS  r(  s         r   is_pinned_defaultr    rx  r&   zself: jt_all, other: jt_allc                 @    US   R                   US   R                   :H  $ r   )r|   r  s      r   is_same_size_defaultr    s     7==DGMM))r&   c                   ^^ [        XUSS9u  pVUR                  S5      mUS   S L =(       d1    [        US   [        [        45      =(       a    [        US   5      S:H  nU(       a  U " TR                  40 UD6nUR                  SS5      (       aS  [        U[        [        45      (       a  [        U5      " U4S jU 5       5      nU$ UR                  TR                  5      nU$ US   n	[        US   [        [        45      n
U
(       d  U	/n	[        TR                  5       U	U TR                  5      u  nnnnU
(       d  US   nXS'   U(       a  TR                  b  [        U S	35      eSS
KJn  U(       a  U(       a6  U " TR                  40 UD6nUR                  SS5      (       a
  U" S U5      nU$ U(       a  [        U S35      eUR                  S5        U
(       a  TR                  /OTR                  nU " TR#                  U5      4SU0UD6$ U(       a  [        U S35      eU " TR                  40 UD6n[%        T5      mUR                  SS5      (       dT  [        US   [        [        45      (       a  US   OUS   /nU H%  nUTR                  S-
  :  d  M  TS==   S-  ss'   M'     U" U4S jU5      $ )NTr   r   r#   r   keepdimFc              3   X   >#    U  H  oR                  TR                  5      v   M!     g 7fr   )r  rR   )r   or   s     r   r   #_apply_reduction.<locals>.<genexpr>3  s     JcCOO < <cs   '*zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapc                 $    U R                  S5      $ )Nr   )r  )r  s    r   rM   "_apply_reduction.<locals>.<lambda>_  s    Qr&   zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsr   rR   c                    > [        U 40 TD6$ r   r	   )r  
out_kwargss    r   rM   r    s    ,q"?J"?r&   )r   r   r   r   r   rb   r   r   r    r  rR   r>   r#   rQ   r(   torch.utils._pytreer  r  r   )rh   	func_nameidentity_elementri   rj   r   r   full_reductionoutdim_to_convert
is_dimlistconverted_dimra  rb  reduce_on_non_batchr  dim_to_passdimlistr   r   r  s                      @@r   rt  rt  #  s   &TMA ..
!C  &$. :e$udm4TZ=N9OST9T  3;;-*->>)U++#t}--3iJcJJ 
 mmCOO4
  &NJu-t}=J() 		+		 %a(%uCLL4k ; ;
 	

 - s{{1j1C~~i//7=J #" k ": :  NN5!/93??+sK$$%56<GKU  + K K  3;;-*-#C(
~~i// j/%?? 5! '( 
 s**}-2-  ?EEr&   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r&  rS  r(  s         r   sum_defaultr    rx  r&   z3self: jt_all, dim: any?, keepdim: any?, dtype: any?c                 $    [        U SS/UQ70 UD6$ )Nr   r   rs  r  s      r   sum_dim_IntListr    s    
 D%<T<V<<r&   z"self: jt_all, dim0: any, dim1: anyc           
      (   [        XUSS9u  p4SSKJn  UR                  S5      nU" UR	                  5       US   US   45      u  pxXvR
                  :X  d  XR
                  :X  a  US:X  d  US:X  a  [        S5      eXvR
                  :X  a  Un	OUn	[        U5      n
XS	'   [        UR                  5       R                  [        [        UR                  5      XvR
                  5      [        [        UR                  5      XR
                  5      5      40 U
D6$ [        UR	                  5       US   UR
                  S
5      US'   [        UR	                  5       US   UR
                  S
5      US'   [        U " UR                  40 UD640 [        U5      D6$ )NTr   r   r   r   dim0dim1z?Transpose is not supported on the batch dimension for jagged NTrR   	transpose)r   r   r   r   r#   rR   rc   r   r
   r{  r  r   rb   r|   r0   r   )rh   ri   rj   r   r   r   r   r  r  to_dimrJ  s              r   transpose_intr
    su    'TMA 6
..
!C"3779z&/A:fCU.VWJD
 $//"919	Q  ??"FF#C(
$*=!JJL""#C		ND//J#C		ND//J

 
 	
 *	:f%sJv *	:f%sJv S[[7J7O>#;NOOr&   zself: jt_all, dims: anyc                    [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U5      n[        UR                  5      nU[        U5      :w  a  [        SU S3S[        U5       S3-   5      eS	S
KJn	  U	" X5      n
[        U
5      [        [        U
5      5      :w  a  [        S5      eUR                  b  [        S5      eU
S	   S	:w  a  [        S5      eU
R                  UR                  5      US'   U
SS   Vs/ s H  n[        XUR                  5      PM     nnXS'   [        U " UR                  40 UD640 UD6$ s  snf )NTr   r   r8   z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().r   r   z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTrR   r   )r   r   r   rb   r|   rc   r   r   setrQ   indexrR   r   r
   r   )rh   ri   rj   r   r   r   r8   rJ  inp_dimr   canonicalized_dimsr#   
inner_dimss                r   permute_defaultr    sv   &TMA ..
!C>>&!D$J#))nG #d)CG9BOQRUVZR[Q\\^_`
 	

 6*79
#c*<&=">>EFF
||P
 	
 !!K
 	
 !3 8 8 IJ} &ab))C 	G#//:)   $vS[[7J7F:FFs   	 Ec                   ^^	 [        XUSS9u  p4UR                  S5      mUR                  S5      m	TR                  S:w  a>  [        TR                  5      [        T	5      :w  a  [        STR                   ST	 S35      e[        T	5      S	:  d  [        TT	5      (       d  [        S
TR                   ST	 35      eUU	4S jn[        [        T	5      S-
  5       Vs/ s H
  oe" U5      PM     nn[        R                  " TR                  5       5         [        U " TR                  U5      40 [        T5      D6sS S S 5        $ s  snf ! , (       d  f       g = f)NTr   r   r   r   zVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (r  r  zview(): cannot view shape z as c                 p   > U TR                   S-
  :X  a  TR                  R                  U 5      $ TU S-      $ rP   )rR   r   r   )	inner_idxr   r   s    r   get_inner_size$view_default.<locals>.get_inner_size  s8    !++;;##I..	A&&r&   )r   r   rR   r   r|   r(   rb   r   r  rI   inference_modeis_inferencer
   r   r   )
rh   ri   rj   r   r   r  ro   
inner_sizer   r   s
           @@r   view_defaultr    s2   
 'TMA ..
!C>>&!D
!cii 0E$K ? YYKtfB@
 	
 4y1}.sD997		{$tfMNN ' .33t9q=-AB-A.#-AJB 
		c..0	1Dj9Q^C=PQ 
2	1	 C 
2	1s   E&E
EzHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anyc                 &   [        XUSS9u  p4UR                  S5      nUR                  5       S::  a  [        S5      eUS   nUR                  UR
                     nUR                  5       [        U5      -
  nUS:X  a  [        S5      eXv;   a  UR                  b  [        S	5      eXv;   Ga:  [        R                  R                  R                  UR                  R                  UR
                  S
9UR                  /UR                  /S9n	[        R                  R                  R                  [        R                   " UR                  R                  S   S4UR"                  UR$                  S9UR                  /UR                  /S9R'                  U	R                  5      n
UR                  R)                  5       R+                  S5      R+                  S5      U	R                  S   -  n[        R,                  " U	SSS9U-  nX-
  U
-  n[        R,                  " [        R.                  " U5      SSS9U-  n[        R0                  " XS   -   5      nX-  n[        R                  R                  R3                  UUR                  /UR                  R                  S   S9R5                  SUR                  UR
                  S-   S  5      n[7        U40 [9        U5      D6UU4$ U " UR                  40 UD6u  np[7        U40 [9        U5      D6X4$ )NTr   r   r   zOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsr   r   zVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr   )rW  r   )r;  dtype)r   r   )r#   r  r   rZ  rD   )r   r   r#   r(   r   rR   rb   rQ   rI   r  r  r]  r   r   r   r^  onesr;  r  r   r  r  r   squaresqrtr`  	unflattenr
   r   )rh   ri   rj   r   r   r   r   ragged_sizenum_dims_not_normalizedpadded_inputpadded_maskragged_lengthsmeanpadded_normalizedvariancestdpadded_layer_normjagged_layer_norm_valuesr%   s                      r   native_layer_norm_defaultr.  )  s   
 'TMA ..
!C
wwyA~]
 	
 ""45))COO,K!ggi#.>*?? 	 1$d
 	
 &3<<+Cy
 	

 	'yy~~EEKK//    \\N) F 
 iinnDDJJ))!,a0399U\\N) E 
 &
	 	 LL))!,66q9L<N<Nq<QQ 	
 II
  	 
 II./
  	 jju$556-3#(99>>#Q#Q\\NKK%% $R $
 )		#//A-/0
 	! 1I^C5HI
 	
 S[[7J7FD7>##67CCr&   zpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      nU " UR                  UR                  40 UD6u  pxn	Uc  S X4$ [        U40 [	        U5      D6X4$ )NTr   rh  r   rk  )
rh   ri   rj   r   r   rh  r   d_inputd_gammad_betas
             r   "native_layer_norm_backward_defaultr3    s    
 'TMA ~~j)H
..
!C#H$4$4ckkPZPGfg&&8N3$78'JJr&   z"self: jt_all, dim: any, index: anyc                    [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  SSS9u  US'   nU(       a  UR                  5       US      $ UR                  b  [        S5      e[        U5      nUS   UR                  S	-
  :  a  US
==   S	-  ss'   [        U " UR                  40 UD640 UD6$ )NTr   r   r#   selectr  r  zSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesr   rR   )r   r   r0   r#   rR   r!  rQ   rc   r   r
   r   )rh   ri   rj   r   r   r   r/   r  s           r   
select_intr6    s    &TMA ..
!C,<	:e$cooxQU-)Ju) zz|Jw/00
||a
 	

  $J%3??Q..=!Q&!S[[7J7F:FFr&   z7self: jt, dim: any?, start: any?, end: any?, step: any?c                     [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  S5      US'   [        U " UR                  40 UD640 [        U5      D6$ )NTr   r   r#   slice)r   r   r0   r#   rR   r
   r   r   r(  s         r   slice_tensorr9    ss    
 'TMA ..
!C(	:e$coowJu S[[7J7O>#;NOOr&   z8input: jt_all, indices: any, values: t, accumulate: any?c           	         [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U5      UR                  5       ::  d   e[        U5      UR                  S-   :  Ga:  UR                  5       (       d  [        S5      eSSKJn  UR                  nUR                  n	U	n
UR                  R                  UR                  S-
     nU
c  Un
/ UR                  S UR                   QU
PUR                  UR                  S-   S  Q7nUR                  SUS	9nU" U " X40 UD6UR                  UR                  UUU	S
9nU [        R                   R"                  R$                  R&                  :X  a+  UR                  R)                  UR+                  5       5        U$ U$ UR-                  5       c  UR/                  5       R1                  5       nOUR-                  5       n[        R2                  " [        R4                  " XeR                     U:  5      S5        UR/                  5       US      XeR                     -   nUSUR                   U/-   XeR                  S-   S  -   nU [        R                   R"                  R$                  R&                  :X  a  U " UR                  U40 UD6Ul        U$ [7        U " UR                  U40 UD640 [9        U5      D6$ )NTr   r   indicesr   z[index_put(): If ragged dimension is not part of indices, this only works on contiguous NJTsr   r  r  r   z7Some indices in the ragged dimension are out of bounds!r   )r   r   rb   r#   rR   is_contiguousr(   r   r   r   r   r   r   r  r   rI   r  r  
index_put_r  rM  r{  r   r   r  _assert_asyncr   r
   r   )rh   ri   rj   r   r   r   r;  r   r   r   r   r   r  
padded_inpnew_njtr   ragged_indicesfunc_indicess                     r   r=  r=    s    'TMA #w/C nnY'Gw<3779$$$
7|coo))  ""m  	6**
**
!++##COOa$78"L
YY()

 YYs*,-

 ))#<)H
$3
3LL!!
 599>>,,444KKgnn./J
 {{}++-$$&++-			'//*W45A [[]71:.1IIN 	COO$
	 //A%'
(		)  uyy~~((0003;;C
C
S[[,5*5

 r&   z{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anyc                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ r&  r'  r(  s         r   convolution_defaultrD  )  sJ     'TMA ..
!CS[[7J7O>#;NOOr&   c                 r   [        XUSS9u  p4US   n[        UR                  5       US   SUR                  5      u  p6pxU(       a  U(       d  U(       a   eUS   n	SUS'   [	        [
        R                  R                  R                  R                  SS40 UD6n
UR                  b  UR                  OUR                  R                  5       n[        U
R                  5       S-
  5       H  nUR                  S	5      nM     X-  nU	(       d  UR                  UR                  5      nU$ S
n[	        U SU40 UD6$ )NTr   r   r#   r(  r  r   r   rD   gzG?)r   r>   r#   rR   rt  rI   r  r  r   dim_IntListrQ   r   r  r  r  r   )rh   ri   rj   r   r   r   ra  rb  r  r  intermediate_sumr   r  intermediate_values                 r   mean_dimrI  8  s1    'TMA W
CBS	5	C?Q) &&&Y' $
9+IINN**FA
9C

 #&,,":#,,@Q@Q@S'++-12A''+G 3(++coo.C

 D&*<K
KKr&   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r&  rS  r(  s         r   mean_defaultrK  a  rx  r&   z&self: jt_all, dim: any?, keepdim: any?c                 $    [        U SS/UQ70 UD6$ )Nr7   Frs  r  s      r   any_dimsrM  l  s    D%@@@@r&   z%self: jt_all, dim: any, keepdim: any?c                     [        XUSS9u  p4US   /US'   [        [        R                  R                  R
                  R                  40 UD6$ NTr   r#   )r   rM  rI   r  r  r7   r8   rh   ri   rj   r   r   s        r   any_dimrQ  q  M    &TMA
 $E*+JuEIINN&&++:z::r&   c                 $    [        U SS/UQ70 UD6$ )Nr   Trs  r  s      r   all_dimsrT  |  s    D%????r&   c                     [        XUSS9u  p4US   /US'   [        [        R                  R                  R
                  R                  40 UD6$ rO  )r   rT  rI   r  r  r   r8   rP  s        r   all_dimrV    rR  r&   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r&  rS  r(  s         r   all_any_max_min_defaultrX    s;     'TMA ..
!C*z**r&   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   minr   rI   finfor  maxrt  rh   ri   rj   r   r   	dtype_maxs         r   min_dimr`    O    &TMA Jw/556::ID%DTDVDDr&   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   r]  r   rI   r\  r  rZ  rt  rh   ri   rj   r   r   	dtype_mins         r   max_dimrf    ra  r&   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   aminr[  r^  s         r   amin_defaultri    Q     'TMA Jw/556::ID&)EdEfEEr&   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   amaxrc  rd  s         r   amax_defaultrm    rj  r&   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   argminr[  r^  s         r   argmin_defaultrp    Q     'TMA Jw/556::ID(IGGGGr&   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   argmaxrc  rd  s         r   argmax_defaultrt    rq  r&   c                 &   [        XUSS9u  p4UR                  S5      nU H{  n[        U[        5      (       d  [	        S5      eUR                  5       US   R                  5       :w  a  [	        S5      e[        XeS   R                  5      (       a  Mr  [	        S5      e   [        US   R                  5       S-   US	   US   R                  S
5      US	'   [        U " U Vs/ s H  ofR                  PM     sn40 UD640 [        US   5      D6$ s  snf )NTr   r  z+stack(): expected all nested tensors inputsr   z9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurer   r#   r  )r   r   r   r
   r(   r#   r   r   r0   rR   r   r   )rh   ri   rj   r   r   r  rV   s          r   stack_defaultrv    s   &TMA
 nnY'G!\**LMM557gajnn&&K  "!QZ%5%566X   )
1j/1G1GJu )Aii)8Z8<J7ST:<V )s    DzQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?c                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " XeR                  40 UD640 [	        U5      D6$ )NTr   r;  r   r'  )rh   ri   rj   r   r   r;  r   s          r   embedding_defaultrx    s`    
 'TMA
 nnY'G^^H%FV__3
37Eg7N r&   zYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      nU " UR                  UR                  40 UD6$ )NTr   r;  r,  rS  )rh   ri   rj   r   r   r;  r,  s          r    embedding_dense_backward_defaultrz    sQ    
 'TMA nnY'G../K##W__C
CCr&   c                 r    [        XUSS9u  p4UR                  S5      nUR                  R                  5       $ r&  )r   r   r   detachr(  s         r   values_defaultr}  "  s<     'TMA ..
!C ;;r&   c                 b    [        XUSS9u  p4UR                  S5      nU " UR                  5      $ r&  rS  r(  s         r   all_defaultr  5  s4    &TMA ..
!Cr&   z-self: jt_all, padding: any, output_size: any?c                    [        XUSS9u  p4UR                  S5      nUR                  b  [        S5      eUS   nUb  XeR                     nO4UR
                  b  UR                  OUR                  R                  S5      nUR                  5       nUR                  S:  a  UR                  UR                  S-
  S5      nUR                  n	UR                  5       S:  a  UR                  SS	9nO%UR                  5       S:X  a  UR                  S
5      nUR                  [         R"                  L n
U
(       a0  UR$                  (       a  UR'                  [         R(                  5      n[         R*                  R,                  R/                  UUR0                  /U/US   5      nU
(       a0  UR$                  (       a  UR'                  [         R"                  5      n[3        U	5      S:  a  UR5                  S
U	SS  5      nO [3        U	5      S:X  a  UR7                  S
5      nUR                  S:  a  UR                  UR                  S5      nU$ )NTr   r   z?to_padded_tensor(): not supported for nested tensors with holesr  r   r   r   r  rD   padding)r   r   rQ   r(   rR   _max_seqlen_tensorr^  r   r   r{  r  r   r#   r   r  r  rI   boolis_cudar?  halfr  r  r]  r   rb   r"  r   )rh   ri   rj   r   r   r   r  max_seq_lenr{  values_shapeis_bool
padded_outs               r   to_padded_tensor_defaultr  @  s   
 'TMA ..
!C
||M
 	

 ]+K!//2 %%1 OO!!!$ 	 ZZ\F
!!#//A"5q9<<Lzz|a!,		!!"% llejj(G6>>5::&??		9	J :%%]]5::.
 <1))"l12.>?
	\	a	''+

))#//1=
r&   zcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?c                    [        XUSS9u  p4US   US   peUR                  SS5      nUS:  a  UR                  US5      nUR                  nUR	                  5       S:  a  UR                  SS	9nO%UR	                  5       S:  a  UR                  S
5      nUR                  [        R                  L n	U	(       a0  UR                  (       a  UR                  [        R                  5      n[        R                  R                  R                  XV/US   5      n
U	(       a0  U
R                  (       a  U
R                  [        R                  5      n
[!        U5      S:  a  U
R#                  S
USS  5      n
O [!        U5      S:  a  U
R%                  S
5      n
US:  a  U
R                  US-
  S5      n
US   nUS   n0 nUb  XS'   Ub  XS'   ['        U
UUUS9$ )NTr   paddedr   r4   r   r  r   r  rD   r   r   r   r   )rR   r   )r   r   r  r   r#   r   r  r  rI   r  r  r?  r  r  r  r`  rb   r"  r   r
   )rh   ri   rj   r   r   r  r   r4   padded_ragged_dim1_shaper  r{  r   r   metadata_caches                 r   "_nested_from_padded_tensor_defaultr    s   
 'TMA !*Jy,AGa0J A~!!*a0%||zz|a!,		!!"% llejj(G6>>5::&YY^^;;	:g.F 6>>5::& #$q(!!"&>qr&BC	%	&	*#A~!!*q.!4L)JL)JN'1|$'1|$&	 r&   zcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c                     [        XUSS9u  p4US   US   US   pvnUS   nUS   n	US   n
0 nU	b  XS'   U
b  XS'   [        UUUUUS	9$ )
NTr   r   r   r   r4   r   r   )r   rR   r   )r   r
   )rh   ri   rj   r   r   r{  r   r   r4   r   r   r  s               r    _nested_view_from_jagged_defaultr    s    
 'TMA
 	799 F
 L)JL)JL)JN'1|$'1|$& r&   c                 V    [        XUSS9u  p4UR                  S5      nUR                  $ r&  )r   r   r   r(  s         r   _nested_get_offsetsr    /    &TMA ..
!C<<r&   c                 V    [        XUSS9u  p4UR                  S5      nUR                  $ r&  )r   r   rQ   r(  s         r   _nested_get_lengthsr    r  r&   c                 V    [        XUSS9u  p4UR                  S5      nUR                  $ r&  )r   r   rR   r(  s         r   _nested_get_ragged_idxr    s/    &TMA ..
!C??r&   c                 v    [        XUSS9u  p4UR                  S5      nUR                  R                  SS 5      $ )NTr   r   r   r   r   r   r   r(  s         r   _nested_get_min_seqlenr    >    &TMA ..
!C""<66r&   c                 v    [        XUSS9u  p4UR                  S5      nUR                  R                  SS 5      $ )NTr   r   r   r  r(  s         r   _nested_get_max_seqlenr   	  r  r&   zself: jt, mask: anyc                    [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  S:  a  [        S5      eUR                  UR                  :w  a%  [        SUR                   SUR                   35      eUR
                  R                  UR                  5       5      n[        R                  " UR                  5       R                  S	S
9S5      n[        U5      nXR                     US'   [        SSU0UD6$ )NTr   r   rp  r   z3masked_select only support 2-D selections currentlyzMask with shape z& is not compatible with input's shape r   rY  r  r   r{  r3   )r   r   r   r(   r   r   masked_selectr{  r  r  r  r   r   r
   )	rh   ri   rj   r   r   r   rp  
res_valuesmask_cumsums	            r   masked_select_defaultr  	  s    &TMA ..
!C>>&!D
xx!|PQQ	djj	 tzzl*PQTQZQZP[\
 	
 **4;;=9J%%,,,3V<K#D!,,/DO 
 r&   z2grad_output: t, self: jt_all, dim: any, index: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        R                  " XVR                  S9nUR                  US   US   5      R                  U5        U$ )NTr   r   r,  )r  r#   r  )r   r   rI   
zeros_liker  r5  rM  )rh   ri   rj   r   r   r   r,  
grad_inputs           r   _nested_select_backward_defaultr  $	  sv    
 'TMA ..
!C../K!!#->->?Jj'G)<=CCKPr&   zself: jt_all, s: anyc                     US   nUS   nU " UR                   U5        U " UR                  U5        UR                  b  U " UR                  U5        g g r   )r   r   rQ   )rh   ri   rj   r   streams        r   record_stream_defaultr  6	  sM    
q'C!WFfv
||S\\6"  r&   zRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?c                     [        XUSS9u  p4UR                  S5      n[        US   5      S:X  a  U " UR                  40 UD6$ [	        S5      e)NTr   r   r   r   z2new_empty() not supported for NJT with shape != ())r   r   rb   r   r(   r(  s         r   new_empty_defaultr  A	  sY     'TMA ..
!C
:f!#CKK.:..
K
LLr&   zself: jt_all, ...c                 j    [        S U 5       5      n[        U " S U 5       0 UD640 [        U5      D6$ )Nc              3   T   #    U  H  n[        U[        5      (       d  M  Uv   M      g 7fr   rT   r   s     r   r   &activation_backward.<locals>.<genexpr>c	  s     Ldsjl.Kssdr   c              3   h   #    U  H(  n[        U[        5      (       a  UR                  OUv   M*     g 7fr   )r   r
   r   r   s     r   r   r  f	  s%     TtZ\::ckkCts   02r   )rh   ri   rj   r,  s       r   activation_backwardr  V	  sH     LdLLKTtT	
	

 
% r&   zself: jt_all, value: anyc                 h    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6  U$ r&  rS  r(  s         r   fill__Scalarr  m	  s;    &TMA ..
!C#
#Jr&   )flex_attentionflex_attention_backward)GraphModulequerykeyr  	score_mod
block_maskscalekernel_optionsscore_mod_other_buffersmask_mod_other_buffersc	                    U R                  5       S:X  a(  UR                  5       S:X  a  UR                  5       S:X  d   e[        S Xx-    5       5      (       a  [        S5      e[        U R	                  5       R                  S5      UR	                  5       R                  S5      UR	                  5       R                  S5      UUUUUUS9	n	[        R                  R                  U	S   R                  SS5      R                  S5      U R                  U R                  U R                  U R                  S9R                  SS5      n
[        R                  R                  U	S   R                  SS5      R                  S5      U R                  U R                  U R                  U R                  S9R                  SS5      nX4$ )	N   c              3   |   #    U  H2  n[        U[        R                  5      =(       a    UR                  v   M4     g 7fr   )r   rI   rJ   r  )r   bufs     r   r   flex_njt.<locals>.<genexpr>	  s.      CC 	3%7#--7Cs   :<zflex_attention(): Nested tensor score_mod / mask_mod buffers are not currently supported. Please file an issue if this is important to you.r   )r  r  r  r  r  r  r   r   r   r   )r#   r7   r(   flex_attention_hopr{  r  rI   r  nested_tensor_from_jaggedr  r   r   rQ   r   r   )r  r  r  r  r  r  r  r  r  r%   
output_njtlogsumexp_njts               r   flex_njtr  	  s    99;!	Q599;!3CCC  *C   U
 	
    #

q!  #% 75
F 77q	Aq!))!,**** 8  i1o  LL::q	Aq!))!,**** ;  i1o  &&r&   r  	logsumexprh  grad_logsumexpfw_graphjoint_graph.c                    [        U R                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UUU	U
UUUS9nUu  nnnn[        R                  R                  UR                  SS5      R                  S5      U R                  U R                  U R                  U R                  S9R                  SS5      n[        R                  R                  UR                  SS5      R                  S5      UR                  UR                  UR                  UR                  S9R                  SS5      n[        R                  R                  UR                  SS5      R                  S5      UR                  UR                  UR                  UR                  S9R                  SS5      nUUUU4$ )Nr   )r  r  rh  r  r  r  r  r  r  r  r  r   r   r  )flex_attention_backward_hopr{  r  rI   r  r  r  r   r   rQ   r   r   )r  r  r  r  r  rh  r  r  r  r  r  r  r  r  r%   dense_q_graddense_k_graddense_v_gradscore_mod_other_buffer_grads
njt_q_grad
njt_k_grad
njt_v_grads                         r   flex_njt_backwardr  	  s   & )  #

q!  #JJL""1%""$..q1",,Q/%,,.88;% 75F$ NTJL,.J77q!$,,Q/**** 8  i1o  77q!$,,Q/(((( 8  i1o  77q!$,,Q/**** 8  i1o  
J0LMMr&   z	self: anyc                     SSK Jn  U" 5       $ )Nr   )_nt_view_dummy)$torch.nested._internal.nested_tensorr  )rh   ri   rj   r  s       r   _nested_get_jagged_dummyr  	  s    Cr&   r  IMPLr  CPUCUDAMeta)F)TFF)r   )r3   r3   )r   r  r   typingr   r   rI   torch.nn.functionalr[  r\  r  torch.fx.operator_schemasr   torch.nested._internal.sdpar   r   r
   r   Any__annotations__r   Dictr   r0   r>   strru   r}   r   r   r   r   register_jagged_funcCallabler   r   r   r   r  r  r  r  r  r  r#   r
  r	  r  r  r  primr:  r  r   r  r<  r  r  cloner$  linearr)  linear_backwardr5  r?  r  r7  _to_copyrK  rM  rN  r|  
empty_like	ones_liker  
randn_likerQ  zero_rT  _softmaxrf  _softmax_backward_datari  native_dropoutrn  native_dropout_backwardrq  r  dim_intru  rw  r_   rJ   r}  r  r  r  r  r  r!  r  r  r   r  r  r  r  r  r/  r  bmmr  r   r  r  r  r  r  wherer.  r  _pin_memoryr  	is_pinnedr  is_same_sizer  rt  r   r  rF  r  r  r
  permuter  view_unsafe_viewr  native_layer_normr.  native_layer_norm_backwardr3  r5  r6  r8  r9  	index_putr=  convolutionrD  r(  rI  rK  r7   r8   rM  rQ  r   rT  rV  r]  rZ  rX  r`  rf  rh  ri  rl  rm  ro  rp  rs  rt  r  rv  	embeddingrx  embedding_dense_backwardrz  r{  _nested_get_valuesr}  r  r  r  _nested_from_padded_tensorr  _nested_view_from_jaggedr  r  r  r  r  r  r  r  _nested_select_backwardr  record_streamr  	new_empty	new_zerosnew_onesr  elu_backwardhardshrink_backwardhardsigmoid_backwardhardtanh_backwardsoftplus_backwardsoftshrink_backwardr  fill_Scalarr  &torch._higher_order_ops.flex_attentionr  r  r  r  torch.fx.graph_moduler  py_implTupler_  r  Unionr  r  library_scoped_libraryimplr3   r&   r   <module>r     s       !    8 K ' c #% $sCx. %3. :Y8=S =D =@

#&
+7
AD
	
0. !((8HI &HX,> &RKS\E$P 		33;;		''		""		$$		  ((		!!))		))11 00, eiinn++33^D E 
YY^^  !
	
 eiinn22::NKF LF( 	IINN  ..0S
 	IINN  "FEE2 eiinn++335UVP WP 	IINN""**<	> eiinn''--/JKP LP eiinn--55~F% G%P 	IINN  "Q$ UYY^^**22N C
 		!!))		  ((		!!))		!!))	 PP eiinn**22NC D 	IINN##%QEPEPP 	IINN))11=	 	IINN!!))+N

 	IINN**22+			 	IINN8>	> eiinn))113NO+ P+ 	IINN!G" 	IINN##++-T" 	IINN!!#P77  eiinn**224VW(
 X(
V eiinn++//1JKE LEP eiinn,,002FGK HK eiinn..668PQE RE, eiinn((002JK L0 eiinn++335OPM QM` eiinn((002KLE ME  	IINN!!#LNN  eiinn..668LMK NK eiinn1199;TU V" eiinn66>>O P> 	IINNI$ eiinn0088:RSP TP eiinn..668PQ+ R+ 	IINN'')F**gFT eiinn((002MN+ O+ 	IINN""9=	= 	IINN  "F'P'PT eiinn,,446OP%G Q%GP 
YY^^  %)).."="="E"EF,R	,R^ 	IINN$$,,N^D	^DB 	IINN--55v
K	
K eiinn++//1UVG WG8 	IINN=
P	
P 	IINN$$> 	IINN%%>Q		Qh 	IINN&&G
P
P 	IINNR#L#LL eiinn))113NO+ P+ eiinn((--/WXA YA eiinn((,,.UV; W; eiinn((--/WX@ Y@ eiinn((,,.UV; W; 		""		""		""		""	 ++ eiinn((,,.UVE WE eiinn((,,.UVE WE 	IINN!IFF 	IINN!IFF 	IINN!!#KHH 	IINN!!#KHH eiinn**224LM N< 	IINN$$W	 	IINN++33_D	D 		%%		))11 	 	  eiinn((00.A B 	IINN##++38	8v 	IINN--55i2	2j 	IINN++33i	8 eiinn88@@.Q R eiinn88@@.Q R eiinn;;CC^T U eiinn;;CC^T7 U7 eiinn;;CC^T7 U7 eiinn22::<QR S0 	IINN**228	 eiinn22::<RS# T# 		  ((		  ((		''
 Y
M
M 		##++		**22		++33		((00		((00		**22 
	
	 eiinn**113MN O . L) &($&5'<<5'	5' <<5' 	5'
 5' 5' cN5' #5' "5' 5<<%&5' *5'p $$\2 &($&;N<<;N	;N <<;N 
	;N
 ||;N ll;N LL;N Hk)*;N ;N ;N ;N cN;N #;N ";N 	LL%,,eHU\\4JC4O.PP;N 3;N~ eiinn==EE{S T ]]""662dII(*BEJII(*BFKII(*BFK 322s   ^=A__
A_