
    Bj                     %   U d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 d dlmZ d dlZd dlmZ d dlmZ d dlZd dlmZmZmZ d dlmZmZmZmZmZ d dl m!Z! d dl"m#Z# d d	l$m%Z% d d
l&m'Z' d dl(m)Z) d dlm*Z*m+Z+m,Z,m-Z- d dl.m/Z0 d dl1m2Z2m3Z3 ddl4m5Z5m6Z6 ddlm7Z7m8Z8m9Z9  e
d          Z: ed          Z;ej<        j=        ej<        j>        z  Z?e	e@d<    ejA        eB          ZCejD        jE        ZEejD        jF        ZFejD        jG        ZGejD        jH        ZHejD        jI        ZI eg eEjJ        eEjK        eEjL        eEjM        eEjN        eEjO        eEjP        eEjQ        eEjR        eEjS        eEjT        eEjU        eEjV        eEjW        eEjX        eEjY        eEjZ        eEj[        eEj\        eEj]        eEj^        eEj_        eEj`        eEja        eEjb        eEjc        eEjd        eEj/        eEje        eEjf        eEjg        eEjh        eEji        eEjj        e'eEjk        eEjl        eEjm        eEjn        jo        eEjp        jq        eEjr        eGjs        eHjt                  Zui  e            euZveEjw        eEjx        eEjy        eEjz        j{        eEj|        eEj}        eEj~        eEj        eEj/        eEj        eEj        eEj        eEj        eEj        eEj        j        eEj        eEj        eEj        eEj        eEj        eEj        eEj        j        eEj        eEj        eEj        eEj        j        eEj        eEj        eEj        gZeej<        j        ej<        j>        z           e@d<    eeve           de?ee?         z  deee;e:f         gee;e:f         f         fdZ eeEj        j        g          dej        dej        dedej        fd            Z eeEj        j        g          dej        dej        dej        dej        fd            Z eeEj        g          dej        dej        d eod!eod"edej        fd#            Z eeEj/                  d$ej        d%ej        dej        dz  d&ej        dz  d'edeej        ej        ej        f         fd(            Z eeEj        j{        g          ddd)d*ej        d+ej        j        dz  d,ej        j        dz  ddfd-            Z eeEj        g          e	 	 dd.ej        d+ej        j        dz  d,ej        j        dz  dej        fd/                        Z eeEj        g          ed.ej        dej        fd0                        Z eeEj        g          d1eeoej        z           d2ej        j        d3edej        fd4            Z eeEj        g          dd5d.ej        d6eod7ej        d8ej        d9ej        j        dej        fd:            Z eeEj        j{        g          d1eeoej        z           d;eeo         d3edej        fd<            Z eeEj        g          dej        d$ej        dej        d=eeo         d>eoeeo         z  d?eoeeo         z  d@eoeeo         z  dAedBeeo         dCeodDee         deej        ej        ej        f         fdE            Z eeEj        j        g          dd.ej        dFeodej        fdG            Z eeEj        g          e	 ddHej        dIej        dJej        dz  dej        fdK                        Z eeEj        g          e	 	 	 ddHej        dLej        dMej        dJej        dz  dNej        j        d9ej        j        dej        fdO                        Z eeEj        g          e	 ddHej        dPej        dJej        dz  dej        fdQ                        Z eeEj        j{        g          	 ddReej                 d6eodej        fdS            Z eeEj        g          d.ej        dej        fdT            Z eeEj        g          dd5d.ej        dUej        d9ej        j        dz  dej        fdV            Z eeEj        g          dHej        dej        fdW            Z eeEj        eEj        g          dHej        dej        fdX            Z eeEj        eFj        g          dHej        dYej        dej        fdZ            Z eeEj        eFj        g          dHej        dYej        dej        fd[            Z eeEj                  	 	 ddHej        d6eodz  d]edej        fd^            Z eeEj                  	 	 ddHej        d6eodz  d]edej        fd_            Z eeEj        g          dHej        d6eodeod`eodej        f
da            Z eeEj        j{        g          dHej        d1eeoej        z           dej        fdb            Z eeEj        j        g          dHej        dcej        dej        fdd            ZdHej        deej        ej        f         fdeZ eeEj                  dddd\d\ej        dfdHej        d2eoez  dcej        dz  dgej        dz  dhej        dz  diedjedkej        dej        fdl            Zddej        dmdnedoej        f         dHej        dcej        dz  dhej        dz  dkej        d3edej        fdpZ ej        eEj        ge!          dHej        d3edej        fdq            Z ej        eEj        ge!          dHej        d3edej        fdr            Z ej        eEj        j{        ge!          dHej        dseod3edej        fdt            Z ej        eEj        j        ge!          dHej        dueodseod3edej        f
dv            Z eeEj        j{                  dseod1eeoej        z           d3edej        fdw            Z eeGjs        j{                  	 dd$ej        dej        d&ej        dz  dej        fdx            Zs eeHjt        j{                  d$ej        dyej        dzej        dej        d{ej        d|ej        d&ej        d}ej        d~ej        deodej        fd            Zt eejD        jG        j                  dej        dej        fd            Z eeEj        g          e	 	 	 ddej        dej        deodeodedej        fd                        Z eeEj        j                  deej                 deej                 dej        j        deej                 fd            Z eeEj        j                  deej                 deej                 deej        j                 deej                 fd            ZeEj        j{                            ej        j        j                   eeEj                  d$ej        dej        d&ej        dz  dej        dz  dej        dz  dedededeej        ej        ej        f         fd                        Ze j        deeedoef         f         fd            Zdeeedoef         f         fdZ eeEj                  dHej        dej        dej        dej        fd            Z eeIj        j                  d$ej        deodeod'edcej        deej        ej        f         fd            Z eeEj                  	 ddHej        d7ej        dej        dedej        f
d            Z eeEj                  	 ddHej        d7ej        dej        dedej        f
d            Z eeEj|        j{                  edej        dej        d6eodej        dej        f
d                        Z| eeEj                  dddHej        d6eod7ej        dej        dededej        fd            Zd.ej        deeo         d>eoeeo         z  dz  d?eoeeo         z  d@eoeeo         z  ded6eodeej        ej        f         fdZ eeEj                  	 	 	 	 dd.ej        deeo         d>eoeeo         z  dz  d?eoeeo         z  d@eoeeo         z  dedeej        ej        f         fd            Z eeEj                  	 	 	 	 dd.ej        deeo         d>eoeeo         z  dz  d?eoeeo         z  d@eoeeo         z  dedeej        ej        f         fd            Z eeEj                  d.ej        deeo         deej        ej        f         fd            Z eeEj        j                  d\d\ddddej        dHej        j        dedededz  dej        dz  dej        fd            Z eeEj        j                  d\d\ddHej        j        dej        dededej        f
d            Z eeEj                  	 	 	 	 ddHej        dej        dedededej        dz  deej        ej        f         fd            Z eeEj        j                  	 ddej        deodz  dej        fd            Z	 	 	 	 	 dd$ej        dej        d&ej        dz  d>eeo         d?eeo         d@eeo         dCeodej        fdĄZdS )    N)Callable)Any	TypeAliasTypeVar)	ParamSpec)core_aten_decompositionsget_decompositionsremove_decompositions)_grid_sampler_2d
_index_addembedding_dense_backwardpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_args)extra_random_decomps)counters)	is_fbcode)	out_dtype)pad_listlike)elementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDsuggest_memory_formattype_to_dtype)native_layer_norm)guard_or_falsestatically_known_true   )configinductor_prims)is_gpu,needs_fallback_due_to_atomic_add_limitationsuse_scatter_fallback_T_P_GenericOperatordecomps_to_excludeopsreturnc                     t          | t                    r| n| gD ]&}|t          v rt                              d|            't          j        | t                    S )Nzduplicate decomp: %s)
isinstancelistdecompositionslogwarningdecompregister_decomposition)r&   ops     `/var/www/html/Carbon-Document/venv/lib/python3.11/site-packages/torch/_inductor/decomposition.pyr/   r/      s\      T**5cc 5 5KK.444(n===    startendweightc                     t          |           }|t          j        k    r|                    |          }|| z
  }|dk    s|dk    rt          j        ||d|z
             S t          j        | ||          S )Nmemory_format      ?g            ?alpha)r   torchcontiguous_format
contiguousadd)r3   r4   r5   fmtdiffs        r1   _lerp_scalarrC      s      
&
&C
e%%%nn3n//;D}}$ydC&L/::::9UD////r2   c                 :   t          |           }|t          j        k    r|                    |          }|| z
  }|                                dk    }d|z
   }t          j        |||          }t          j        |||           }t          j        |||d          S )Nr7   r9   r:   r   )value)r   r=   r>   r?   abswhereaddcmul)	r3   r4   r5   rA   rB   maskneg_omwwbases	            r1   _lerp_tensorrM      s      
&
&C
e%%%nn3n//;D::<<3DfoGD'6**A;tS%((D=q$a0000r2   grad_outputindicesnum_weightspadding_idxscale_grad_by_freqc                 D    | j         rt          S t          | ||||          S N)is_xpuNotImplementeddecomp_embedding_dense_backward)rN   rO   rP   rQ   rR   s        r1   _embedding_dense_backwardrX      s3      *Wk;8J  r2   inputnormalized_shapebiasepsc                 D    | j         rt          S t          | ||||          S rT   )is_mtiarV   decomp_native_layer_norm)rY   rZ   r5   r[   r\   s        r1   _native_layer_normr`      s+     } #E+;VT3OOOr2   )minmaxsymbolra   rb   c                    d S rT    )rc   ra   rb   s      r1   sym_constrain_range_for_sizerf      s	     Fr2   xc                 b    ||                      |          } ||                     |          } | S rT   )	clamp_min	clamp_max)rg   ra   rb   s      r1   clamprk      s5     KK
KKHr2   c                 Z    | d|                                                                  z   z  S )Nr   )negexp)rg   s    r1   siluro      s$     AEEGGKKMM!""r2   size
fill_valuekwargsc                     |                     d          }|2t          t          |                    |d<   t          j        | |fi |S t
          S )Ndtype)getr   typer=   fullrV   )rp   rq   rr   rt   s       r1   rw   rw      sS     JJwE}'Z(8(899wz$
55f555r2   r;   dimindextensorr<   c                    t                      s| j        t          j        k    rt          S t          | |||d|          S )NF)inplacer<   )r   rt   r=   bfloat16rV   r   )rg   rx   ry   rz   r<   s        r1   	index_addr~     s@     ;; M17en44!S%eLLLLr2   physical_layoutc                 \    t          |          t          t          t          |                              k    }|rt          j         fi |S dgt                     z  }t          |          D ]
\  }}|||<   t          j         fd|D             fi |                    |          S )Nr   c                      g | ]
}|         S re   re   ).0lrp   s     r1   
<listcomp>z"empty_permuted.<locals>.<listcomp>/  s    ===DG===r2   )r*   rangelenr=   empty	enumeratepermute)rp   r   rr   is_identitypermpr   s   `      r1   empty_permutedr   !  s     ''4c/6J6J0K0K+L+LLK W{4**6***sSYYo.. 	 	DAqDGG{====_===HHHHPPQUVVVr2   
bias_sizesstridepaddingdilation
transposedoutput_paddinggroupsoutput_maskc                 b   |
d         rt          | j        j                  st          S t                              | dgt          t          d|                                                     z             }t          	                    | |||||||||	|
d         |
d         dg          \  }}}|||fS )N   r   r   F)
r   devicerv   rV   atensumr*   r   rx   convolution_backward)rN   rY   r5   r   r   r   r   r   r   r   r   	grad_biasgrad_inpgrad_weight_s                  r1   r   r   2  s     q> (:(?!@!@ qcDq+//:K:K1L1L,M,M&MNNI#88	QQ/   Hk1 k9--r2   decimalsc                 R    d|z  }t                               | |z            d|z  z  S )Ng      $@r:   )r   round)rg   r   ten_pow_decimalss      r1   	round_decr   S  s/    X~::a**++s5E/EFFr2   selfbatch2r   c                 x   t          | j        d         dk              st          |j        d         dk              r| |z                                  S t          j        r| j        j        dvr}t          | j        d         dk              st          |j        d         dk              rA|                     d          |                    d          z                      d          }|S | j        j        dk    rt          | 	                    d          dk              rt          |	                    d          dk              rnt          d         dxx         dz  cc<   t          j        |                     d          |                    d          z  dd	
                              d          S t          S )Nr   r   cpumpsrx   r   inductordecompose_bmmTrx   keepdim)r   shaper?   r   coordinate_descent_tuningr   rv   	unsqueezer   rp   r   r=   squeezerV   )r   r   r   outs       r1   bmmr   Y  s    TZ]a/00 ,4IQ15 5 , v))+++ ' DK,<N,R,R A!!344 	8MLOq 9
 9
 	 >>"%%(8(8(;(;;@@Q@GGCJ{5   1!233 	8MKKOOq 9
 9
 	 Z 111Q61119Q&.."4"44!T  ill r2   mat1mat2betac                    |j         j        dvrt          |                    d          dk              rwt          |                    d          dk              rQt          |                    d          dk              r+t          d         dxx         dz  cc<   ||z  }||z  || z  z   S | j         j        dk    r|t          |                    d          dk              rt          |                    d          dk              ryt          d         dxx         dz  cc<   t          j        |                    d          |                    d          z  dd	                              d          }||z  || z  z   S t          |                    d          dk              rt          |                    d          d
k              rkt          |                    d          d
k              rEt          d         dxx         dz  cc<   |j
        |z                      dd	          }||z  || z  z   S t          S )Nr   r   r   r   r   decompose_addmmr   Tr      )r   rv   r   rp   r   r=   r   r   r   r   TrV   )r   r   r   r   r   r<   r   s          r1   addmmr   z  s?    {~--!$))B--1"455	-%diilla&788	- &diilla&788	-
 Z !2333q8333+C3;,,{5   1!233 	-8MIIbMMQ9
 9
 	- Z !2333q8333)Q$,,r"2"224  ill  3;,,!$))A,,!"344	-tyy||r122	- tyy||r122	-
 Z !2333q83336D=%%!T%::C3;,,r2   input2c                 \   t           j        r| j        j        dvr{t	          | j        d         dk              st	          |j        d         dk              r?|                     d          |                    d          z                      d          S t	          |                     d          dk              r t	          |                     d          dk              rt	          |                    d          dk              r| j        j        dvs| j        j        dk    rt	          |                     d          dk              rt	          |                    d          dk              rj| j	        |j	        k    rZt          t          j        |           t          j        |          z   dk              r t          d	         d
xx         dz  cc<   | |z  S | j        j        dk    rt	          |                     d          dk              rt	          |                    d          dk              rnt          d	         d
xx         dz  cc<   t          j        |                     d          |                    d          z  dd                              d          S t          S )Nr   r   r   r   r   r   r       r   decompose_mmTr   )r   r   r   rv   r   r   r   r   rp   rt   r   r=   numelr   r   rV   )r   r   r   s      r1   mmr     s    ' HDK,<N,R,R A!!344 	H8MLOq 9
 9
 	H NN1%%(8(8(;(;;@@Q@GGG 	diimmq011!!$))A,,!"344! "&++a..A"566!
 ;>11K%%%diillQ&677 &%fkk!nn&9:: & v|++D 1 1EK4G4G GBNOO , Z 000A5000&= {5   1!233 	8MKKOOq 9
 9
 	 Z 000A50009Q&.."4"44!T  ill r2   tensorsc                    dt           j        dt          ffd}t          t	          ||                     t                    dk    rYt          | dt          j        id         }d         }||j	        k    r|
                                n|                    |          S dt                    cxk     rt          |           k     r#n n t          j                                      S t                    dk    rt          fdD                       rd         }t          |j                  }dk     rt          |j                  z   n|                    t                                |                              j        |                     dz             
                                S t*          S )	Nrg   r'   c                     t          | j                  dk    r t          | j        d         dk              rdS t          | j                  k     r t          | j                 dk              rdS dS )Nr   r   FT)r   r   r   )rg   rx   s    r1   non_empty_tensorzcat.<locals>.non_empty_tensor  si    " qw<<1
a!@!@5QW.1B"C"C5tr2   r   type_promotion_kindr   rt   c              3   ,   K   | ]}|d          u V  dS )r   Nre   )r   tfiltered_tensorss     r1   	<genexpr>zcat.<locals>.<genexpr>  s?       ) )%&a  ) ) ) ) ) )r2   )r=   Tensorboolr*   filterr   r   r   DEFAULTrt   clonetor   catdefaultallr   insertr   expandflattenrV   )r   rx   r   promoted_dtype
filtered_tinpr   r   s    `     @r1   r   r     s   
EL T      2 F#3W==>>
!!+
 ? G
 
  &a(
 !111 ^44	

 
S!""	1	1	1	1S\\	1	1	1	1	1x 0#666 q  S ) ) ) )*:) ) ) & &  q!SY&)AggcC	NN""3S#.//000(s}}S!!(%088cAgFFLLNNN r2   c                    |                                  rWt          j        t          j        | j                  t          d          t          j        | j        | j                            S t          | t          j
                  \  }}t          j        t          j        || j                  }t          j        | dk     |d          }t          j        t          j        |           t          d          |          S )Nnan)r   rt   r   r   g        )
is_complexr=   rG   isnanrealfloatatan2imagr   r   INT_TO_FLOATscalar_tensormathpir   )rg   r   rt   r   rets        r1   angler     s    ||~~ 
{Kuu{1616/J/J
 
 	
 "	;H  HAu 
	TWE!(	C	C	CB
+a!eR
%
%C;u{1~~uU||S999r2   yc                   t          j        |           o|                                 }t          j        |          o|                                }|r|st          S dt           j        dt
          fd}d}| j        dk    r|j        dk    rd}| j        dk    r|                     d          } |j        dk    r|                    d          }|}|||z  }t          j        | j	        |j	                  } ||           s ||          rt          S dt           j        dt           j        fd}	| dz   } |dz   } |	| 
                    | j        j	                            }
 |	|
                    |j        j	                            }t          j        |
|z   d	
          
                    |          }|r|d         S |S )Nrz   r'   c                 X    | j         dk    rdS |                                 d         dk    S )Nr   Fr   r   )ndimr   )rz   s    r1   _requires_fallbackzadd.<locals>._requires_fallback0  s,    ;!5}}r"a''r2   Fr   Tr   c                     | j         ^ }}|dz  dk    rt          d          g ||dz  dR }|                     |          }|S )zNReshape tensor from [*initial_dims, last_dim] to *initial_dims, last_dim/2, 2]r   r   zQThe size of the last dimension must be even to reshape it to [..., last_dim/2, 2])r   AssertionErrorview)rz   initial_dimslast_dim	new_shapereshaped_tensors        r1   reshape_tensor_complexz#add.<locals>.reshape_tensor_complexI  sj     #),x a<1 c  
 6l5HM5155	 ++i00r2   )	start_dim)r=   	is_tensorr   rV   r   r   r   reshapepromote_typesrt   r   r   r   )rg   r   r<   x_is_complex_tensory_is_complex_tensorr   output_size_zerozcomplex_typer   
x_reshaped
z_reshapedresults                r1   r@   r@   #  s     /!,,?/!,,? &9 (5< (D ( ( ( ( v{{qv{{v{{IIaLLv{{IIaLL	AAI&qw88L!  2 21 5 5 u|     $ 	
AA	AA''qv|(<(<==J''qv|(<(<==J]:
2bAAAFF|TTF ayMr2   c                 <    |                                  rt          S | S rT   )r   rV   r   s    r1   conj_physicalr	  g  s     Kr2   c                     | S rT   re   r  s    r1   liftr  n  s    Kr2   otherc                 `    t          j        t          j        |          || k    z  | |          S rT   r=   rG   r   r   r  s     r1   fminr  s  )    ;u{5))UT\:D%HHHr2   c                 `    t          j        t          j        |          || k     z  | |          S rT   r  r  s     r1   fmaxr  x  r  r2   Fr   c                 h    | j         t          j        k    rt          j        | ||          S t          S Nr   )rt   r=   r   anyrV   r   rx   r   s      r1   amaxr  }  0     zUZy38888r2   c                 h    | j         t          j        k    rt          j        | ||          S t          S r  )rt   r=   r   r   rV   r  s      r1   aminr    r  r2   lengthc                 l    t          j        | |||                              t           j                  S )Nr7   )r=   narrowr   r>   )r   rx   r3   r  s       r1   narrow_copyr    s7     <c5&1177- 8   r2   c                 \    t                               | |                                          S rT   )r   r   r   )r   rp   s     r1   view_copy_defaultr!    s$    
 99T4  &&(((r2   rt   c                 P    |                                                      |          S rT   )r   r   )r   rt   s     r1   view_copy_dtyper#    s     
 ::<<U###r2   c                      t          j                   \  }} fd|D             }dgt          |          z  }t          |          D ]
\  }}|||<   ||fS )Nc                 *    g | ]}j         |         S re   )r   )r   r   r   s     r1   r   z/_get_shape_permutation_like.<locals>.<listcomp>  s    444qTZ]444r2   r   )utils3compute_elementwise_output_logical_to_physical_permr   r   )r   r   r   r   permutationr   r   s   `      r1   _get_shape_permutation_liker)    sw     RSWXXOQ4444O444E#E

"K/**  1A;r2   )rt   layoutr   
pin_memoryrequires_gradr8   r*  r   r+  r,  r8   c          	         || j         n|}|| j        n|}|| j        n|}|t          j        k    r6t          j        | j        ||||||          }|                    |          S |t          j        k    sJ t          |           \  }	}
t          j        |	||||||          }|
t          t          t          |
                              k    r|S |                    |
                                          S )N)rt   r*  r   r+  r,  r7   )rt   r*  r   r=   preserve_formatrw   r   r   stridedr)  r*   r   r   r   r   )r   rq   rt   r*  r   r+  r,  r8   r  r   r(  s              r1   	full_liker0    s     -DJJUE"NT[[F"NT[[F---J!'
 
 
 yy}y555 &&&&8>>{!'
 
 
 $uS%5%5667777M~~k**00222r2   )rt   r   r8   rand_fn.c                   ||j         n|}||j        n|}|t          j        k    r% | |j        f||d|                    |          S t          |          \  }} | |f||d|}|t          t          t          |                              k    r|S |
                    |                                          S )Nr   r7   )rt   r   r=   r.  r   r   r)  r*   r   r   r   r   )	r1  r   rt   r   r8   rr   r   r(  r  s	            r1   
_rand_liker3    s      -DJJUE"NT[[F---wJ

 
 	
 

 "="
)
)	* 5T::E;W  	 F d5[!1!1223333>>+&&,,...r2   c                 2    t          t          j        | fi |S rT   )r3  r=   randr   rr   s     r1   	rand_liker7    s    ej$11&111r2   c                 2    t          t          j        | fi |S rT   )r3  r=   randnr6  s     r1   
randn_liker:    s    ek4226222r2   highc                 d    t          t          j        t          j        j        d|          | fi |S Nr   r3  	functoolspartialr   randintlow)r   r;  rr   s      r1   randint_likerC    s.    i'(8!TBBDSSFSSSr2   rB  c                 d    t          t          j        t          j        j        ||          | fi |S rT   r>  )r   rB  r;  rr   s       r1   randint_like_lowrE    s0     i'(8#tDDdUUfUUUr2   c                 4    t          j        j        d| |fi |S r=  )r   rA  rB  )r;  rp   rr   s      r1   rA  rA  !  s#     <AtT44V444r2   c                     t           j        j                            |          }t           j        j                            | |||                                d                   S r=  )r=   r&   
_quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weightrp   )rY   r5   r[   packed_weights       r1   #linear_dynamic_fp16_unpacked_weightrL  *  sO     I(MMfUUM9AA}dFKKMM!$4  r2   input_scaleinput_zero_pointweight_scaleweight_zero_point	out_scaleout_zero_pointout_channelc
           	          t           j        j                            ||||          }
t           j        j                            | |||
|||	          S rT   )r=   r&   rH  _wrapped_linear_prepack#_wrapped_quantized_linear_prepacked)rY   rM  rN  r5   rO  rP  r[   rQ  rR  rS  rK  s              r1   wrapped_quantized_linearrW  6  s\     I(@@/ M 9CC  r2   packedc                     dt           j        dt           j        fd} || dddf                   } || ddd f                   }| dd df                             t           j                  |z  |z   S )Nu8r'   c                 &     fddD             \  }}}}t           j        dk    r7||dz  z   |dz  z   |dz  z                       t          j                  d         S |dz  |dz  z   |dz  z   |z                       t          j                  d         S )Nc              3   f   K   | ]+}d |f                              t          j                  V  ,dS .N)r   r=   int32)r   nrZ  s     r1   r   zPq_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32.<locals>.<genexpr>T  s9      GGQbajmmEK00GGGGGGr2   )r   r   r      little   r      r]  )sys	byteorderr   r=   float32)rZ  rg   r   r  rK   s   `    r1   bitcast_u8_to_f32z=q_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32S  s    GGGG,GGG
1a=H$$aLAG,R8>>u}MMiXX"Wb)Q!V4q8>>u}MMiXXr2   .i)r=   r   r   rf  )rX  rg  scalesoffsetss       r1   "q_embedding_bag_byte_unpack_decomprk  Q  s    Yel Yu| Y Y Y Y vc2b5j122FsBCCx 011G#ss(u}--6@@r2   agridinterpolation_modepadding_modealign_cornersc                     | j         t          j         d          k    o%|dk    o|                     t          j                   }t	          | |||||          }|S )Nr   r   r7   )rm  rn  ro  rp  _expand_grid)r   r=   is_contiguousr>   decomp_grid_sampler_2d)rl  rm  rn  ro  rp  rr  outputs          r1   grid_sampler_2drv  _  sv     	
EL''' 	C!#	COO%*AOBBL $	-!#!  F Mr2   start_tensorsend_tensorsc           	          t           j                            | t           j                            t           j                            ||           |                    S rT   )r   _foreach_addList_foreach_mulScalar_foreach_sub)rw  rx  r5   s      r1   _foreach_lerp_scalarr    sP     !!  "";>>	
 	
  r2   scalarsc           	          t           j                            | t           j                            t           j                            ||           |                    S rT   )r   rz  r{  r|  
ScalarListr~  )rw  rx  r  s      r1   _foreach_lerp_scalarlistr    sP     !!$$"";>>	
 	
  r2   running_meanrunning_vartrainingexponential_average_factorepsilonc           
          t                               | |||||||          \  }}	}
|r||	|
fS ||                    d          |                    d          fS )Nr   )r   native_batch_norm	new_zeros)rY   r5   r[   r  r  r  r  r  rl  bcs              r1   miopen_batch_normr    s|     $$"	 	GAq!  1ay	 r2   c                  "    i t           t          S rT   )r+   r   re   r2   r1   fast_random_decompsr    s    5n5 455r2   c                      t           j        rt          S t           j        r;t                              t
          j        j        j        j	        d           t          S t                      } | S )z"decomps can change based on configN)r   fallback_randomr+   "fallback_embedding_bag_byte_unpackpopr=   r&   	quantizedembedding_bag_byte_unpackr   r  )r  s    r1   select_decomp_tabler    sV     0 59.HPRVWWW ""FMr2   rI   sourcec                    ddl m}m}  || j        |j                  rt
                              | |g          \  } }|                    d                              d          dz
  }d | ||fD             \  }}}t
          	                    |||gd          }	t          j        ||	|                              | j                  S t          S )Nr   )BackendFeaturehas_backend_featurer   r   c              3   >   K   | ]}|                                 V  d S rT   )r   )r   rg   s     r1   r   z!masked_scatter.<locals>.<genexpr>  s*      ,W,WQQYY[[,W,W,W,W,W,Wr2   )codegen.commonr  r  r   MASKED_SCATTER_WITH_INDEXr   broadcast_tensorsr   cumsum_unsafe_masked_indexr=   rG   r   r   rV   )
r   rI   r  r  r  
source_idx	self_flat	mask_flatsource_flatr  s
             r1   masked_scatterr    s     DCCCCCCC4;(PQQ J ++T4L99
d\\"%%,,Q//!3
,W,W4vBV,W,W,W)	9k**;	J<QRSS{9fi88==djIIIr2   	quant_min	quant_maxc                    t          j        |           \  }}||z
  t          ||z
            z  }t          j        |t          j        |g                    }|t          j        ||z                                t           j                  z
  }t          j        |||          }|                    t           j	                  |                    t           j
                  fS rT   )r=   aminmaxr   rb   r   r   r   intrk   float64int64)	rY   r  r  r\   rt   min_valmax_valscale
zero_points	            r1   choose_qparams_tensorr    s     }U++GWw%	I(=">">>EIeU\3%0011EU[599<<UYGGGJZI>>J88EM""JMM%+$>$>>>r2   
accumulatec                     |                                  }t          j        ||g|                    |j                  |          }|                    | j                  S rT   )r   r=   	index_putr   r   )r   ry   r  r  	flatteneds        r1   putr    sS     IE7FNN5;77 I TZ(((r2   c                 h    t                               | |||          }|                     |          S )N)r  )r   r  copy_)r   ry   r  r  r   s        r1   put_r    s-     ((4:(
>
>C::c??r2   ru  input_dtypec                     | |z  }t          j        ||d          }t          j        | ||          }| j        |k    r|                    |          }|                                S )NTr   )r=   r   r   fmart   r   r?   )rN   ru  rx   r  new_grad_outputsum_new_grad
grad_inputs          r1   _softmax_backward_datar    sm     "F*O9_#tDDDL#VG\?KKJ K'']];//
  """r2   Tinclude_selfsrcreduction_typer  c                   |dk    rt          | j                  s| j        j        p| j        j        }t	          j        |          }|r,| }t	          j        |                               |||          }	nZ|                     ||d          }t	          j        |                               |||          }	|		                    |	dk     d          }	|                    |||          }|r||	z  n||	z  S t          t          j        j        || j        |j        |j        j        d          rt           S | j        |dz   d                                          | j        d |                                         z  }
|                                g| j        |dz   d          | j        d |         R }g t'          | j        |z
  | j                  dt'          d| j        |z
            R }|                    t          j                                      |
                              |                              |          }|                     |||||          S )Nmeanr   r   Tr  )r    rt   is_floating_pointr   r=   	ones_liker~   
index_fill
zeros_likemasked_fillr!   r   scatter_reduce_twor   rv   rV   r   r   r   r   r   r  repeat_interleaver   r   scatter_reduce)r   rx   ry   r  r  r  true_divisiononesr   countsrepeatsindex_shaper   scatter_indexs                 r1   index_reducer  "  sG    (T
) ) 
4M
8Ms## 	7C_T**44S%FFFF//#ua00C%d++55c5$GGF''
A66FmmC,,,?sV||#-? 
	
    jq#))++dj#.>.D.D.F.FFG;;==L4:cAgii#8L4:dsd;KLLKNU49s?DI..NNE!TY_4M4MNND		7	#	#					  !    r2   kernel_size	ceil_modec                 F   |dk    rdg|z  }|dk    rdg|z  }|s|}t          ||          }t          ||          }t          ||          }t          ||          }t          j        t          j        |          }t
          j        j                            ||          s'|t          j	        t
          j
                  j        k    rt          S t                              | |||||          \  }}	t                              |	|| j        | d          |||          }
||
fS )Nr   r   )n_dim)r   r?  reduceoperatormulr=   	_inductorlowering%should_fallback_max_pool_with_indicesiinfoint8rb   rV   prims!_low_memory_max_pool_with_offsets'_low_memory_max_pool_offsets_to_indicesr   )rg   r  r   r   r   r  rx   window_sizevalsrj  rO   s              r1   _max_pool_with_indicesr  X  sC    1}}39!||#)  {C00KHc**H7C((G&#&&F"8<==K 	 FFs 	G 	
 	
 UZ00444;;	 MD' ;;	 G =r2   c           	      .    t          | |||||d          S )Nr   r   r  rg   r  r   r   r   r  s         r1   max_pool2d_with_indicesr    *     "	;9!   r2   c           	      .    t          | |||||d          S )Nr`  r   r  r  s         r1   max_pool3d_with_indicesr    r  r2   output_sizec                 2   | j         ^ }}}|\  }}|dk    s|dk    r>g |||}|                     |          |                     |t          j                  fS ||z  dk    r.||z  dk    r%||z  ||z  g}t                              | |          S t          S )Nr   r   )r   	new_emptyr=   r  r   r  rV   )	rg   r  batchh_inw_inh_outw_outo_sizer  s	            r1   adaptive_max_pool2dr    s     UD$LE5zzUaZZ'5'%''{{6""AKKekK$J$JJJe|qTE\Q..u}dem4++A{;;;r2   	out_int32rightsidesortersorted_sequencer  r  r  r   c                    t                               | t          j        |g| j                  ||||          d         S )Nr   r  r   )r   searchsortedr=   rz   r   )r  r   r  r  r  r   s         r1   searchsorted_scalarr    sQ     dVO$:;;;    	 	r2   r  r  
boundariesc                    t                               t          j        | g|j                  |||                              d          S )Nr  r  r   )r   	bucketizer=   rz   r   r   )r   r  r  r  s       r1   bucketize_scalarr
    sL     >>dVJ$5666	   
 gajjr2         ?UUUUUU?noiselowerupper	generatorc                     |rW| dk    }t                               | |||          }t          j        || |z  |           }t          j        ||d          }	||	fS ||z   dz  }
t                               | |
          t          j                    fS )Nr   )r  r   r   )r   uniformr=   rG   
leaky_relur   )r   r  r  r  r  r  not_positiverru  	noise_outnegative_slopes              r1   rrelu_with_noise_functionalr    s      EqyLLueyLAA\4!8T::Ka33	y  %-1,t^44elnnDDr2   repeatc                    t           j        j        rt          S |t	          |          t
          urt          S | j        j        dk    rt          S | j        t          j	        t          j
        fv sJ | j        dk    sJ |                     d          }t          j        || j                  }t          j        ||| j        t          j	        k    d          }t          j        ||                     d          dz
            S )Nr   r   r   r  Tr  )rb   )r   tritonautotune_at_compile_timerV   rv   r  r   rt   r=   r^  r  r   r  aranger  rk   rp   )r  r  r  posrO   s        r1   repeat_interleave_Tensorr    s    
 }-  d;//s::}U""<EK55555;!]]1F
,{6=
9
9
9C  ;D  G ;wFKKNNQ$67777r2   r   r  c           	      |   |                                  dk    r|                                 dk    s
J d            |d         }|d         }|d         }|                     d          }|                    d          }t          j                            ||||df|df|df|          }	|	                    d          S )Nr`  z,Expect (N,C_in,L) and (C_out,C_in//groups,K)r   r   r   )r   r   r   r   )rx   r   r   conv2dr   r   )
rY   r5   r[   r   r   r   r   input_2d	weight_2dout_2ds
             r1   conv1d_to_conv2dr&    s     99;;!

 1 1 16 !2 11
 AYFajG{H r""H  $$I [  {!A !  F >>"r2   )NNrT   )Nr   r   )NF)r   r   F)F)Nr   r   F)r  r  FN)Nr   r  r   r   )r?  loggingr   r  rd  collections.abcr   typingr   r   r   typing_extensionsr   r=   torch._decomp_decompr.   torch._prims_common_prims_commonr&  $torch.ao.quantization.fx._decomposedr   r	   r
   torch._decomp.decompositionsr   rt  r   r   rW   r   r   $torch._decomp.decompositions_for_rngr   torch._dynamo.utilsr   torch._environmentr   !torch._higher_order_ops.out_dtyper   torch._inductor.utilsr   r   r   r   r   torch._refsr   r_   %torch.fx.experimental.symbolic_shapesr   r    r   r   r   r    r!   r"   r#   _opsOperatorBaseOpOverloadPacketr$   __annotations__	getLogger__name__r,   r&   r   r  r  rH  quantized_decomposed_adaptive_avg_pool2d_backwardindex_selectaddmvr  bitwise_and_bitwise_or_
clamp_min_distelu
empty_likeflipgeluhardtanhlcmr  linalg_vector_norm_log_softmax max_pool2d_with_indices_backward_native_batch_norm_legit#_native_batch_norm_legit_functional$_native_batch_norm_legit_no_training_batch_norm_with_update"_batch_norm_with_update_functional_batch_norm_no_updatebatch_norm_backwardr  native_group_normnll_loss2d_backwardpermute_copyrrelu_with_noise_backward_softmaxsin_sqrt__to_copytril_indicestriu_indicesunbind_copyr  upsample_bilinear2dvechann_windowrL  rW  inductor_decompositionsr+   _unsafe_indexr  #_unsafe_masked_index_put_accumulate+_scaled_dot_product_flash_attention_for_cpur   r  rj   ri   r~   gluselect_scatterslice_scatterro   splitr   r   r   unbindbaddbmmrH   addcmul__foreach_addcmulr}  _foreach_addcmul_addcdivaddcdiv__foreach_addcdiv_foreach_addcdiv_lerplerp_r%   r*   
OpOverloadr/   r   rC   rM   r   rX   	ShapeTypetupler`   rf   SymInttypesNumberrk   rw   r   r   r   r   r   r   rt   r   r   r   r   r@   r	  r  detach_r  r  r  r  r  	view_copyr!  r#  
StrideTyper)  r0  r.  r*  r   r8   r3  r7  r:  rC  	low_dtyperE  rA  r  rk  rv  _foreach_lerpr  r  r  r  py_impl_CDispatchKeyAutogradcachedictr  r  r  choose_qparamsrz   r  r  r  r  strr  r  r  r  r  r  r	  r
  r  	Generatorr  r  r&  re   r2   r1   <module>r     s/!           



 $ $ $ $ $ $ * * * * * * * * * * ' ' ' ' ' '        # # # # # # + + + +         
              F E E E E E ( ( ( ( ( ( ( ( ( ( ( ( 7 7 7 7 7 7 . . . . . .            F E E E E E W W W W W W W W $ $ $ $ $ $ $ $          WT]]Yt__#j5
8SS ) S S Sg!!y~	I	Y!
y5 ,,,*,, 	
, 		,
 	, 	, 	, 		, 	, 	, 		, 		, 	, 	, 	,  	!," 	#,$ 	-%,& 	%',( 	0),* 	1+,, 	$-,. 	//,0 	"1,2 	 3,4 	5,6 	7,8 	9,: 	 ;,< 	=,> 	&?,@ 	A,B 		C,D 	
E,F 	G,H 	I,J 	K,L 	M,N 	O,P 	 $Q,R 	S,T 	5U,V 	+W,. . ^ K,,..J2IJ
 	,4<NN!NHIJLHKLLM LM IJ=Q D.1LLM   B  n&8 9 9 9>	D!12	2>xB (2r6"223> > > > )*++0 05< 0 05< 0 0 0 ,+0 )*++1<1#l149L1
\1 1 1 ,+1  6788\  	
  \   98" .//
P<
Po
P L4
P ,
	
P
 

P 5<u|34
P 
P 
P 0/
P :BCDD &*%)	  L 
	d	" 
	d	"	
 
   ED %%# &*%)	 	|			d	"	 
	d	"	 \		 	 	 $# &%	 $$#EL #U\ # # #  %$# $$	
sU\!
"	"	 	 \		 	 	 %$	 ()) !"M M M|M	M <M L	M ;M \M M M *)M* ,4566W
sU\!
"W#YW W \	W W W 76W  2344..<. L. S		.
 $s)O. 49_. DIo. . I. . d. 5<u|34. . . 54.@ ,-..G G G GU\ G G G /.G
 
## %) 
,L {T! \	    $#> %%
 %)  !# #
,#
,# ,# {T!	#
 +
# ;# \# # #  &%#L 	"" %)& &
,&L& {T!& \	& & &  #"&Z )*++ : :%,:	: \: : : ,+:z %%:U\ :el : : : &%:& 
##
 (,	@ @ @|@|@ ;$	@
 \@ @ @ $#@F +,--     .- DL122u|     32 EJ/00Iu| IEL IU\ I I I 10I EJ/00Iu| IEL IU\ I I I 10I 	""  
,	t  \	   #" 	""  
,	t  \	   #" )*++

,
	
 
 	

 \
 
 
 ,+
 /011)
,)
sU\!
") \) ) ) 21) -.//$
,$;$ \$ $ $ 0/$
 
,
 
5?E,,-
  
  
  
  ''
 !%"&"&).)>)3 )3 )3
,)3e)3 ;	)3
 L4)3 L4)3 )3 )3 &)3 \)3 )3 )3 (')3` !%"&).)>/ / /c5<'(/
,/ ;	/
 L4/ &/ / \/ / / /@ /1EFF2EL 2C 2EL 2 2 2 GF2 02FGG3U\ 3S 3U\ 3 3 3 HG3  1 9:<PQQTu| T3 T# T%, T T T RQT  1 ;<>RSSV
,V V(+V7:V
\V V V TSV ,--5
5
sU\!
"5 5 \	5 5 5 .-5 	EMNN !% <L ,
 \	   ON 
;CDD< l L	
 , | , | L  \   ED4 	+EFF
Au| 
A 
A 
A 
A GF
A -.//   |
,  	
  \    0/@ *122
%
el#
 K
 
%,	
 
 
 32
 *566
%
el#
 %+$%
 
%,	
 
 
 76
 ''(<(EFF.//<L ,
 ,%	
 $  !&  5<u|34   0/ GF< 6T#xS'9"9: 6 6 6 6	T#xS'9"9: 	 	 	 	 +,,
,
, L \	   -,$ ,;BCC?<?? ? 
	?
 ;? 5<%&? ? ? DC? !!
 	
) 
)
,
)<
) L
) 	
)
 \
) 
) 
) "!
) 	""
 	 
,< L 	
 \   #" 3;<<##L# 
# 	#
 \# # #  =<#( )** 2 2 2
,2	2 <2 
	2
 2 2 \2 2 2 +*2j5|5c5 $s)Od"5 49_	5
 DIo5 5 
5 5<%&5 5 5 5p 455 &*  !
 
|
c
 $s)Od"
 49_	

 DIo
 
 5<%&
 
 
 65
 455 &*  !
 
|
c
 $s)Od"
 49_	

 DIo
 
 5<%&
 
 
 65
 011|"&s)
5<%&   21" )011
 "&	 	 	\	
+
	 		
 	 *	 L4	 \	 	 	 21	& -..
   
+
 	
  \   /. 899 %(,E E
,E<E E 	E
 E %E 5<%&E E E :9E& .566 #8 8L8t8 \8 8 8 7684 !%) )<)L) ,
) #J	)
 3Z) Cj) ) \) ) ) ) ) )r2   