a,jGfUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZmZddlmZmZddlZddlZddlZddlmZddlmcm Z!ddl"m#Z#m$Z$ddl%m&Z&dd l'm(Z(dd l)m*Z+dd l,m-Z-m.Z.dd l/m0Z0dd l1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHddlImJZJddlKmLZLddlMmNZNdd lOm*Z*ddlPmQZQddlRmSZSmTZTmUZUmVZVddlWmXZXddlYmZZZm[Z[m\Z\ddl]m^Z^ddl_m`Z`ddlambZbdd lcmdZdmeZemfZfmgZgmhZhdd!limjZjmkZkmlZlerddlmZnddloZoe*jpZqd"erd#<ejsetZud$erd%<ejvjwZwejvjxZxeGd&d'ZyeGd(d)ZzeGd*d+Z{dd/Z|dd2Z}dd3Z~dd5ZGd6d7Zdd8ZeZdd;Zdd<Zdd?Z dddKZddLZddMZddNZddOZddPZddQZddRZddSZddWZdd\Zdd_Z dddiZddoZddrZddtZdduZddvZddyZdddzZdd{Z dddZ dddZ ddd@d@dddZdddddZedZddZddZddZejddZddZddZd dZd dZd dZd dZd dZddZd dZ dd dZddZddZddZdd„ZddÄZddʄZddlmZdd΄ZddτZ dddфZddӄZddքZdd؄Z dddڜdd܄Z dddZdS() annotationsN) defaultdictdeque)Callable) dataclassreplace)Any TYPE_CHECKING)countersis_node_meta_valid)(create_structured_trace_for_min_cut_info)is_with_effects)config)CustomKnapsackSolverCustomRuntimeEstimator)FakeScriptObject) is_builtin) LazyStringtrace_structured) trace_log)extract_tensor_metadata) BackwardState) is_sym_node py_sym_types) magic_methodsmethod_to_operator)find_symbol_binding_fx_nodes free_symbolsis_symbol_binding_fx_nodeoptimization_hintstatically_known_falsestatically_known_true) graph_drawer) OrderedSet)CheckpointPolicy)GraphInfoProvider) dp_knapsackdp_knapsack_sliding_hirschberggreedy_knapsack ilp_knapsack)KnapsackEvaluator) AOTOutputSavedForBackwardsAOTOutput#SavedForBackwardsNoVcCheckAOTOutput)_is_functional_graph)is_opaque_node)get_aot_graph_name)_is_bwd_seed_offset_is_fwd_seed_offset _is_primal _is_tangentget_cuda_generator_meta_val) fx_graph_cseget_aten_targetraise_getitemsboolAOT_PARTITIONER_DEBUGzlogging.LoggerlogcneZdZUdZded<ded<ded<ded<ded<dd Zdd ZddZddZddZ dS)OpTypesz8Class for keeping track of different operator categorieszOrderedSet[Callable[..., Any]] fusible_opscompute_intensive_ops random_opsview_opsrecomputable_opsnodefx.Nodereturnr;c.t||jvSN)r9r@selfrEs ]/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/torch/_functorch/partitioners.py is_fusiblezOpTypes.is_fusiblehst$$(888c.t||jvSrI)r9rArJs rLis_compute_intensivezOpTypes.is_compute_intensivekst$$(BBBrNc.t||jvSrI)r9rBrJs rL is_randomzOpTypes.is_randomnst$$77rNc.t||jvSrI)r9rCrJs rLis_viewzOpTypes.is_viewqst$$ 55rNc.t||jvSrI)r9rDrJs rLis_recomputablezOpTypes.is_recomputabletst$$(===rNNrErFrGr;) __name__ __module__ __qualname____doc____annotations__rMrPrRrTrVrNrLr?r?^sBB////9999....,,,,44449999CCCC88886666>>>>>>rNr?ceZdZUded<ded<ded<ded<ded<ded <ded <ejdd ZddZddZddZ ddZ dS)NodeInfo list[fx.Node]inputsOrderedSet[fx.Node]_required_fw_nodesrequired_bw_nodestangents_closureunclaimed_nodesdict[fx.Node, int]fw_orderstatic_lifetime_input_nodesrGcJtdjDfdS)Nc3K|]}|VdSrIr].0ns rL z-NodeInfo.required_fw_nodes..s" 0 01Q 0 0 0 0 0 0rNcj|SrI)rh)rnrKs rLz,NodeInfo.required_fw_nodes..s a@PrNkey)sortedrcrKs`rLrequired_fw_nodeszNodeInfo.required_fw_nodess: 0 0/ 0 0 06P6P6P6P    rNrnrFr;c||jvSrI)rcrKrns rLis_required_fwzNodeInfo.is_required_fwsD+++rNc||jvSrI)rdrxs rLis_required_bwzNodeInfo.is_required_bwsD***rNc||jvSrI)rfrxs rL is_unclaimedzNodeInfo.is_unclaimedsD(((rNintcT||jvrtd|d|j|S)NNode z not in fw nodes!)rcAssertionErrorrhrxs rL get_fw_orderzNodeInfo.get_fw_orders7 D+ + + !=!=!=!=>> >}QrNN)rGr`rnrFrGr;)rnrFrGr~) rXrYrZr\ functoolscached_propertyrvryr{r}rr]rNrLr_r_xs++++****))))((((    4444    ,,,,++++))))      rNr_cBeZdZUded<ded<ded<ded<ded<dS) MinCutOptionsr;ban_if_used_far_apartban_if_long_fusible_chainsban_if_materialized_backwardban_if_not_in_allowlistban_if_reductionN)rXrYrZr\r]rNrLrrsN$$$$&&&&!!!!rNrrErFrGch|jddtjtjfvS)N recompute)metagetr%MUST_RECOMPUTEPREFER_RECOMPUTErEs rLmust_recomputers0 9==d + +')0 rNfx_gfx.GraphModulecH|jjD]}t|rdSdS)NTF)graphnodesrrrEs rLhas_recomputable_opsrs7   $   44  5rNc|jjD]F}t|r5t|jdr t jj|jjvrdSGdS)NtagsTF) rrrhasattrtargettorchTagnondeterministic_seededrrs rLhas_recomputable_rng_opsrs^   4   V,,  1T[5EEE44 5rNr~c t|jdtjtjfrdSt|jdtjs*t dt|jddS)Nvalr&z.expected node.meta['val'] to be SymFloat, got ) isinstancerrSymIntSymBoolSymFloatrtypers rL sym_node_sizersw$)E"U\5=$ABBq di& 7 7  UT$)EBR=S=S U U    1rNceZdZddZdS)InvalidNodeBaserGstrcdS)Nz Invalid Noder]rus rL__repr__zInvalidNodeBase.__repr__s~rNN)rGr)rXrYrZrr]rNrLrrs(rNrc6t|jdddkS)N namespace_c10d_functional)getattrrrs rLis_not_collectivers 4; T 2 26H HHrN getitem_nodefx.Node | Nonec:|jtjkrdS|jd}|jd}t |t jr |jdkrdSd|jvrdS|jd}||vrdS||}t |t jr|SdS)zGiven a getitem node, check if it extracts from a higher-order op that has kwargs mapping the key back to an original input. Returns the original input node if found, None otherwise. Nrr& call_functionkwargs) roperatorgetitemargsrfxNodeopr)r ho_resultrsroriginal_inputs rL_get_ho_op_original_inputrs h...t!!$I  A C i ) )Y\_-L-Lty'''t  h 'F &tC[N."'** 4rNc|jtjjjjtjjjjfvrdS|jd}t|tj sdSt|S)zCheck if node is a view/reshape of a higher-order op output that aliases an input. Returns the original input node from the higher-order op's kwargs if the pattern matches, None otherwise. Nr) rropsatenviewdefaultreshaperrrrr)rEsources rL_is_copy_node_bw_onlyrsa  {59>.6 8N8VWWWt Yq\F fbg & &t $V , ,,rNenvdict[fx.Node, Any]ct|}|'||vr#t||ts||St|}|'||vr#t||ts||SdS)aTry to find a valid input replacement for an invalid forward output. This handles cases where a forward output depends on backward nodes but semantically aliases an input. For example, a view of a getitem from a triton kernel that mutates a buffer in backward, or a direct getitem from such a higher-order op. The original input may be a primal or a valid intermediate node already present in the forward graph. N)rrrr)rErrs rL_find_input_for_invalid_outputrs+400N" c ! !3~.@@ ">"".t44N" c ! !3~.@@ ">"" 4rNF joint_graphfx.Graphrar`outputs outputs_descslist[AOTOutput]subgraph str | Noneignore_must_be_in_fw_bwctj}i|D]-}||j}|j|_||<.|jD]}|sHt |r|dkr||vr t|<)t|r|dkr||vr t|<M|vrR|j dkr t|<h|j dkr`tj |j i|j } fd| D} t| r t|<||fd|<|j dkr||fd|<|j d kr  g} t!||D]{\} } t#| tjrE| vrt'd | d t#| t(rd } | jt,jjjjurt | rt7| j d krmt#| j dtjrH| j dvr9t#| j dt(s| j d} | t9| } | | | 6t=d | d| | f| | }|tA| }||jd<d| D|jd<|!|"|S)a Given a graph, extracts out a subgraph that takes the specified nodes as inputs and returns the specified outputs. This includes specifying non-placeholder nodes as inputs. The general strategy is to initialize all inputs with proxies as we encounter them, and trace through the graph, only keeping values which take in valid proxies. Then, all dead code is eliminated. backwardforward placeholderrczg|]7}t|tjt|t8Sr])rrrr)rmxrs rL z6_extract_graph_with_inputs_outputs..QsIa))3q6?33rNc|SrIr]rrs rLrqz4_extract_graph_with_inputs_outputs..Z CFrNget_attrc|SrIr]rs rLrqz4_extract_graph_with_inputs_outputs..]rrNoutputrz couldn't be found in envNr&rz was invalid, but is outputdescczg|]8}t|tjr|jdnd9S) stack_traceN)rrrrr)rmvs rLrz6_extract_graph_with_inputs_outputs..sL''' &027%;%;E =!!!'''rNoutput_stack_traces)#rGraphrnamerr_must_be_in_backward InvalidNode_must_be_in_forwardrpytreearg_tree_leavesrrany node_copyziprr RuntimeErrorrrrrrcopy_rlenrappendrrtupleeliminate_dead_codelint)rrarrrr new_graphrEnew_nodeall_args output_valuesrx_desc replacementoutrs @rL"_extract_graph_with_inputs_outputsrs$ I"$C((33  D !))& $T**  **&&'D $D))  ))&&'D  3;;  W % %#CII W ' '-tyHDKHHH!H 8}} 'D !++D2B2B2B2BCCCII W " "!++D2B2B2B2BCCCII W M-00$$ 6 a ! ! $||"#G1#G#G#GHHH#a&/22 M# H  4 <<<,Q//=AF q(("16!9bg66)q S((&s16!9~GG)#&afQi.K&"@C"H"HK*!((555$%KQ%K%K%KLLL  Q ( ( ( (   # # # #   5// 0 0C$CHV'''''CH "# !!### NN rNctjoWt|jtjjrt|j p|jtjj j kSrI) ris_non_builtin_to_includerrr_ops OpOverloadrr higher_order triton_kernel_wrapper_functionalrs rLrrsP  +  DK!6 7 7 W 4;@W@W.s$ K K$) K K K K K KrNrrr) rrr find_nodesnextiterrrr)r r!rr fwd_outputs bwd_outputsfwd_outputs_descsbwd_outputs_descss rL_extract_fwd_bwd_outputsr/s$ K K 2 = = = J J K K KG* T,$//8/<< = =>>CGG TFS\\)  M *?*+K/**+K%&6&67%o&6&67  %68I IIrN saved_valuesrrNonecV|D]%}|j|kr||dS&dSrI)rremove)r0r saved_values rL_remove_by_namer5sI#  t # #    , , , EE $rNfwd_module_outputs#list[fx.Node] | tuple[fx.Node, ...]ct|}tt|dz ddD]}t||s|dz}n|S)Nr&)rranger)r6idxis rLfind_first_sym_noder=sk  ! !C 3)**Q.B 7 7-a011 a%C E  JrN@-q=rtorch.fx.Graph torch.fx.Nodemaxfloatminpositionc  ||5|tjjjj|f}tjjj|jd|jd<t|jd|jd<dddn #1swxYwY||5|tjjj j|dgdf}tjjj |jddgd|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j|tj f}tjj j |jdtj |jd<t|jd|jd<dddn #1swxYwY||5|tjjj j||f}tjjj |jd||jd<t|jd|jd<dddn #1swxYwY||5|tjjjj|f} tjjj|jd| jd<t| jd| jd<dddn #1swxYwY|| 5|tjjjj| |f} tjjj| jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjj j j| tjfd|d|j} tjj j | jdtj| jd<t| jd| jd<dddn #1swxYwY| S) Nr%r tensor_metar9Tfp8_scale_pos__rr)inserting_afterrrrrabsrrramaxprimsconvert_element_typefloat64 clamp_min reciprocalmulTensorfloat32r) rrErBrDrEabs_node amax_node amax_64_nodeclamp_min_nodereciprocal_nodemul_node scale_nodes rLcalculate_quantization_scalingr]s   t $ $UU&& IN  &'   %y~199$)E:JKK e'>x}U?S'T'T m$ UUUUUUUUUUUUUUU   x ( (WW'' IN  'RD$'(  !&  3 ; ; M% 2$! ! u)@ u@U(V(V }%WWWWWWWWWWWWWWW   y ) )   ** IO 0 8U]++  $)9?#G#O#O N5 !5=$ $ % ,C  e $, , -(                   | , ,   ,, IN $ ,$-  &+Y^%=%E%E  e $c& & E".E   &. . M*                   ~ . .   -- IN % - ".  ',in&?&G&G   &' ' U#/F   '/ / ]+                    / /UU&& IN  %!3''   %y~188   '   e(?x}U?S'T'T m$UUUUUUUUUUUUUUU   x ( ( Y Y(( IO 0 8EM*8(88TY88)  "'!E!M!M M% %-" " *AQVAW)X)X  & Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y sBB22B69B6BE44E8;E8B&II  I &BLL L$BOOO BQ>>RRB2UU #U r\ quant_type torch.dtyperQ clamp_maxc ||5|tjjjj|tjf}tjjj|jdtj|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j ||f}tjj j |jd|jd|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j||f} tjj j |jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjj jj| |f} tjj j| jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjjjj| |fd|d|j} tjjj| jd|| jd<t| jd| jd<dddn #1swxYwY| S)Nr%rrGfp8_quant_pos_rIrJ)rKrrrrNrOrrUrrrrSrTrQr`r) rrEr\r^rQr`rEtarget_node_32scaled_target_nodeclamp_min_scaled_nodeclamp_max_scaled_nodequant_activation_nodes rLperform_quantizationrh4s   z * *   ,, IO 0 8 &-  &+Y_%I%Q%Q Ie em& & E".E   &. . M*                   ~ . .   "00 IN  % *-1  */);)B)B   & (>* * &2I  #E *2 2  .                   1 2 2   % 3 3 IN $ ,$i0!4! ! -2IN,D,L,L  #E *I- - "5)5L ! &u -5 5 "=1                   4 5 5   % 3 3 IN $ ,'3!4! ! -2IN,D,L,L ! &u -y- - "5)5L ! &u -5 5 "=1                   4 5 5   % 3 3 IO 0 8'48(88TY88!4! !  IO 0 8 8%*51:   "5) 5L ! &u -5 5 "=1                 ! s^B&CC C (BFFF1BIII/BL  LL-BOOOtensor torch.Tensorcb|}|}||zdz S)z Calculate the size of a PyTorch tensor in megabytes (MB). Args: tensor (torch.Tensor): Input tensor Returns: float: Memory size in MB i)numel element_size)ri num_elementsrms rLcalculate_tensor_sizerozs2<<>>L&&((L < 'K 88rNlist[torch.dtype]ctjjjddd}d|dD}|S)N!activation_quantization_aten_passallowed_dtypesztorch.bfloat16cjg|]0}tt|dd1S).r9)rrsplit)rmdtypes rLrz&get_allowed_dtypes..s@16u{{3''+,,rN;)r _inductorrpost_grad_fusion_optionsrrv)rss rLget_allowed_dtypesr{s__+D+ c ,--:H:N:Ns:S:SN rNc<t}t|r|jdj|vrdStjjjddd}t|jd}tjjjddds||kStjjjdddr't||kpt||k St||kS)NrFrr size_in_mbdskip_dynamo_guardsquantize_dynamic_shape) r{r rrwrryrrzrror"r!)rErssize_thresholdr}s rLshould_quantizers'))N d # #ty'7'=^'S'Su_+D+ c,'ty'788J ? ! :+  c &&G^++ ? ! : / #& . . G)n,J+J.,HIII J )~)EFF FrNctjjjddd}t t|ddS)Nrrr^ztorch.float8_e5m2rur9)rryrrzrrrv)r^s rLget_quant_typersO'@+ c,+,, 5***3//3 4 44rNrwtuple[float, float]cFtj|}|j|jfS)z Calculate the range of values for a given torch.dtype. Args: dtype (torch.dtype): The input dtype. Returns: tuple: A tuple containing the minimum and maximum values. )rfinforDrB)rwinfos rLcalculate_rangers" ;u  D 8TX rNc |dd}|jd}t}t|\}}t g}g}t |D]\} } | |kr | jddrbtj j j dddrct|| |d | } t|| | |||| } t| s|| n|| n|| 5|tjjjj| |fd | d | j } tjjj| jd || jd <t/| jd | jd<dddn #1swxYwY| | <fdt |D} t1| }||z}|r| d||z| |dz} |dt5| t6ddxxdz cc<dS)Nrr'rsaved_for_quantizationFrr use_scalingTr?rbrIrJrrGcBg|]\}}||Sr])r)rmr<rEposition_to_quants rLrz*quantize_activation_fw..s;+21da&&rNinductor%activation_quantization_fwd_aten_passr&)r(rrrdict enumeraterrrryrrzr]rhrrrKrrrNrOrrrr= update_argrr )rr!rr+r^rQr`tensor_scale_nodessym_scale_nodesrErEr\ quant_nodeoutput_updated_argsr; scale_nodesrs @rLquantize_activation_fwrs     * *1 -F+a.K!!J*:66Iy(*%'O#K00-5-5$ o % %  9==15 9 9# 5%>3c-&& <4E8 24ZIx #:..7&--j9999#**:6666**400  !&!4!4 <D"J/DhDDDD"5""J  <DD Ie,jOE* 6M".66JOM2               +5 h '6? 6L6L 1 2 2C$6K  % 36I#$$6O O  a233444 Z@AAAQFAAAAAs=BG''G+ .G+ c  d|jD}d}|D]O}|jddr0|jd|jd}tjjjdddr||5d|j dd z t fd |D}dddn #1swxYwY||5| tj jjj||f }tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwY||5| tj jjj||f }tj jj|jd |jd |jd <t#|jd |jd <dddn #1swxYwY||5| tj jjj||f }tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwYn||5| tj jjj||fdt+|j z}tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwYt-|jD]$}||kr||kr|||%Qt4ddxxdz cc<dS)Nc(g|]}|jdk |S)rr'r&s rLrz*quantize_activation_bw.. s$JJJ$M1I1I1I1I1IrNrF dequant_typerrr fp8_scale_ fp8_quant_c32K|]}|jk |VdSrIr)rm bwd_input scale_names rLroz)quantize_activation_bw..s<&&%$>Z77"7777&&rNr%rrGdequant_rJr%activation_quantization_bwd_aten_passr&)rrrpoprryrrzrKrrr)rrrNrOrrrdivrTrlistuserskeysreplace_input_withr ) r bw_inputsactivation_noderErr\divided_target_node_32 dequant_nodeuserrs @rLquantize_activation_bwr sJJ%+JJJIOH@H@ 9==15 9 9G @ IMM2 3 3 39==88L%>3c-''? **400!- 0A0A,PR0S0S!SJ!%&&&&)2&&&""J**:66  &+&9&9 <D"L1':''O  <DD Ie,l$(/ ;R',U3;;O(7               **?;;  -2-@-@ *1-z:.A..*:?9K9R9R',U3Z_U5K::*/600F0KE0RSS+/ >               **+ABB  #(#6#6 <D4lC$7$$L  <DD27> !%e, 8O$)%088L%m4               **400  #(#6#6 <D"L1'#di..8$7$$L  <DD Ie,l!%e, 8O$)%088L%m4               TZ__..// @ @<''DO,C,C++D,??? Z@AAAQFAAAAAs^0:C66C: =C: BF44F8 ;F8 BI==J J BL;;L? L? B(PP P fwd_module bwd_modulebwd_module_inputsdict[str, fx.Node]ctddfdtj|tddfdtddfdjd d jd }|D]}d |jvr|t jd d|j}j|5j |j}dddn #1swxYwY|j d}|j |j d|j d<||j d<| |j |tjjjdddrt'jd } | d} t)| D]} t+| s| } njd d jd } | D]~}d|jvrsj| 5j |j} dddn #1swxYwY| j |j | } t-jtddfddS)Nartifactc dddS)N,before_activation_quantization_fwd_aten_passstringrencodingr]r]rNrLrqz5perform_fp8_activation_quantization..dB   rNc4dddSNFT print_outputinclude_strideinclude_deviceprint_readablersrLrqz5perform_fp8_activation_quantization..h#:44tD5  rN metadata_fn payload_fnc dddS)N+after_activation_quantization_fwd_aten_passrrr]r]rNrLrqz5perform_fp8_activation_quantization..qA   rNc4dddSrrrsrLrqz5perform_fp8_activation_quantization..urrNc dddS)N,before_activation_quantization_bwd_aten_passrrr]r]rNrLrqz5perform_fp8_activation_quantization..|rrNc4dddSrrrsrLrqz5perform_fp8_activation_quantization..rrNrr'rrz^fp8_quant_pos_\d+_rrrTrrrrrr9rc dddS)N+after_activation_quantization_bwd_aten_passrrr]r]rNrLrqz5perform_fp8_activation_quantization..rrNc4dddSrrrsrLrqz5perform_fp8_activation_quantization..rrN)rrrr(rrresubrKrrupdatereplace_all_uses_with erase_noderryrrzrrreversedr6r)rrrr!quant_fwd_module_outputsfwd_noderquant_bwd_inputrquant_bwd_module_inputs bwd_input_locbw_inputscaled_fwd_module_outputsscale_bwd_inputs`` rL#perform_fp8_activation_quantizationr\s           :+_===                   */::h:GGJOPQR, 3 3 8= ( ()-r8=AAI!11)<< S S","2">">HM">"R"R S S S S S S S S S S S S S S S$>.9L  ' ' 6 6 6=AO !9 :3?O  0  + +O < < <   ' ' 2 2 2 6+ c-0#'z'7'B'Bm'B'T'T"U"U/3  !899  Hx((  (  %/$4$?$?8$?$L$LQ$O$TUV$W!1 0 0Hx},,%55mDDWW&0&6&B&B &B&V&VOWWWWWWWWWWWWWWW$++HM::: / :+,,,           s$!C;;C? C? !I<<J J riOrderedSet[fx.Node] | Nonec|r d|Dng}d|D}tjjjdddr d|D}|jdd jd }d |jd D}d} |D]} | j|vrt| r| j|vr!t d | jDd | j d<| j dj | j d<d || jj d<| j dj || jj d<d } | rt||||dSdS)Ncg|] }|j Sr]rr&s rLrz2enable_activation_quantization..s;;;t;;;rNci|] }|j| Sr]rr&s rL z2enable_activation_quantization..sCCCd$)TCCCrNrrexclude_primalsFc0i|]}d|jv |j|S)primalsrr&s rLrz2enable_activation_quantization..s/   $ 8R8RDIt8R8R8RrNrr'rci|] }|j| Sr]rr&s rLrz2enable_activation_quantization..s)  4rNrz*Skipping quantization of static input %s: Trrr)rryrrzrrr(rrrr=debugrrwr) r0rrrir!static_input_namessaved_values_namesr6rshould_perform_fp8_quantrEs rLenable_activation_quantizationrs ' ;;:;;;;  DClCCC 6+ c U##   (4   $)444AA!DI!L$.$4$?$?=$?$Q$Q %" , , 9* * *t/D/D *y... F RRR26DI. /(, %(8(>DIn %JN di ( -.F G@D %@P@V di ( -n ='+ $ +  $5       rN)riromit_aot_autograd_runtimesaved_sym_nodessaved_opaque_nodeslist[fx.Node] | Noner%tuple[fx.GraphModule, fx.GraphModule]c  #t||\}} } } |jd} gtt| } |rgngtt | }gtt | }gtt| }gtt| }|g}t|j||zz|z|z| | d|}tj }|dD]}|j s@t|jt||jt||jI|r^t!d|j Dr@t|jt||jt||jt|r&t|j|st#dt%}g}g}|D]S}t'|}|r+||||>||Tt-|j}t/j||D]c}d |jvr t5|jd |z }t7|d D]"}||vr|||#||z}d||||z|sg}g}g}D]}t=|jd t@r||E|jd d r||v||||ztC|#tED]K\}}|#kr@|jd d s%t#d|d#dtCLt|j| |z|z|z|z|z| #fdtGtCtC|ztC|ztC|zDzd|} t|j|z|z|z|z|z|z| | d|}n}t|j| |z|z|z| dtGtCtC|zDzd|} t|j|z|z|z| | d|}tHj%&|| }!tHj%&||}"tNj(dd tS|!|"|||!|"fS)ajExtract forward and backward graph modules from a joint graph. Args: ignore_must_be_in_fw_bw: When True, disables forward/backward placement enforcement in _extract_graph_with_inputs_outputs. Needed when the joint_module is not an original fwd+bwd joint graph (e.g. a backward graph being re-partitioned for dI/dW splitting). omit_aot_autograd_runtime: When True, skips postprocessing that is only needed when the resulting modules will be wrapped in a custom autograd.Function (the AOTAutograd path). This includes: tangent input handling, version-counter check sorting of saved tensors, opaque object (FakeScriptObject) separation, and fp8 activation quantization. Set this to True when the fwd/bwd modules will be executed directly without autograd. r!rr'Nr)rc3K|]>}|jtjjjjuot |jdkV?dSrN)rrrr wait_tensorrrrrls rLroz+_extract_fwd_bwd_modules.."s_) )  H 2>F F "AG !) ) ) ) ) ) rNz'backward_state_inputs must not be emptyrc|jSrIr)ss rLrqz*_extract_fwd_bwd_modules..Js16rNrrsaved_tensor_with_no_vc_checkFzi=z, no_vc_check_start_idx=z, len(saved_values)=c~g|]9}|kr"|tkrt|nt|:Sr])rr/r.)rmr<no_vc_check_start_idxr0s rLrz,_extract_fwd_bwd_modules..sa   ---!c,6G6G2G2G4A666/22   rNrc,g|]}t|Sr])r.rmr<s rLrz,_extract_fwd_bwd_modules..s0+1--rNrr)*r/rr(filterr5r6r4r3r rr distributed is_availablerr5rallrr$raddrr itertoolschainrrrtclearextendrrrrrr:r_lazy_graph_module_make_graph_moduleinductor_configrzr)$r r0rrr!rirrr+r,r-r. placeholders primal_inputstangent_inputsfwd_seed_offset_inputsbwd_seed_offset_inputsbackward_state_inputs bwd_graphdistributed_enabledrE saved_symbolssaved_sym_nodes_bindingsaved_sym_nodes_derivedsymbolsymbol_bindings new_symbolsrsaved_values_with_vc_checksaved_values_no_vc_checksaved_opaque_objectsr< fwd_graphrrrs$ ` @rL_extract_fwd_bwd_modulesr"s4 !OOOCK/1B %00M0BBL7fZ667M'Q-Qvk

> > >$ 25LLMMM $b &("#% !  8 8D$)--..0@AA 8$++D1111>FF 8(//5555*11$777769QQRRR #$> ? ?!..  GAt)))y}}%DeLL(vQvv8Mvvcfgsctctvv7   2 2  " #! !         %%.//0,--./**+     $;+   .7    " #! !   %  % $  $   $;    7   2 2 , & 8 s<0033G3GGHH  $;    7    $ %$ $   $;    &99, RRJ&99, RRJ044 /    '    '      z !!rN)static_lifetime_input_indicesri _joint_inputsr r#list[int] | Nonec g}d}|jjD]1}t|st|st |r|}2|t d|jjD],}t |s||||urn-td|Dt|}t|} |rRt|jd'tj dt||||St|d}t jst%|t'|t)||g}t+|||} g} g} g} t,jd d }d d}d fd }|jjD]x}|jvr |jdkr&|jd|Dvr>|jt,jjjj t,jj!j"j t,jj!j#j t,jj!j$j t,jj!j$j%fvrtM|r| |||r|j'(dtRj*kr.s9$$tw(/B/B /B/B/B/B$$rNrzxTrying to unsafely apply AC to a non-functional graph with the default partitioner. Falling back to min-cut partitioner.)r!r#Tis_default_partitionrErFrGr;cd|jvp6t|jdtjjS)NrGr)rrrr _subclasses FakeTensorrs rL is_tensorz$default_partition..is_tensor#s; ) Z IMM% %"3">. .  rNcntd|jDot|jdkS)Nc3@K|]}|jtjkVdSrI)rrrrmrs rLroz=default_partition..is_multi_output..*s,GGD x//GGGGGGrNr)rrrrs rLis_multi_outputz*default_partition..is_multi_output(s8 GGDJGGG G G $DJ!# rNc|do-|jdvo$ p!|jtjjjjuS)NF) impure_random)rr) is_impurerrrrrrr)rErs rLr6z$default_partition..is_impure.sW NNN / /  ('U;ei&@&L&TT rNrc3 K|] \}}|V dSrIr])rmkrs rLroz$default_partition..Cs73 3 !QA3 3 3 3 3 3 rNrz.Trying to apply AC on a graph with impure op: z, rz Expected z to be a tensorc&g|] }|jv |Sr]r)rmrnforward_node_namess rLrz%default_partition..os&UUUAFBT4T4T14T4T4TrNc34K|]}t|VdSrIrrls rLroz$default_partition..ps(77!{1~~777777rNrrr!ri)is_impure_noder&enable_activation_offloadingFrrW)Arrrr5r4rr6rr$rrr0warningswarn#min_cut_rematerialization_partitioncleanup_recompute_tagsr(unsafe_allow_optimization_of_collectivesforce_save_collectivesforce_save_effectful_opsforce_save_bw_mutation_srcclassify_nodesrrrrr named_modulesrrr_assert_scalarrprofiler_record_function_enter_new_record_function_enter_record_function_exit_RecordFunctionrrrr% MUST_SAVEr1rrr rrrfromkeysr_sync_decision_cross_ranksrir"rrfunctionalize_rng_opsr#reordering_to_mimic_autograd_enginer@,_activation_offloading.activation_offloadingr:thread_graphsafe_rng_from_hopsrd)r r$r!r#ri forward_nodes last_noderEgraph_has_recomputable_opsgraph_has_recomputable_rng_ops node_infor0rrr/r3r6backward_usages fw_module bw_moduler@rr:s @@rLdefault_partitionras@MI"( t $ $  4(8(8 >@@AAOdmm,>??DDFFGG (T1,2DlSS "*&/&K#3'-'$? IyO''7H'III O''7H'III!C ) #8iC4H4H$$ Iy8 BB *         %$    '    y))Iy))I.yeLLLI 9 &''!++29$OOO i rNg.Arlc||jzSrI)itemsize)rlrws rL_tensor_nbytesrds 5> !!rNcjddd|jvr|jd}t|trdSt|ttfrt fd|DSt|t r-t fd |DSt|tj r |Std t|d ||j d ks"|j tjjjjurd Std|d)NrobjectrGr~ct|tjsdStt |d|jS)Nrfallback)rrrTrdr rlrwrs rL object_nbytesz_size_of..object_nbytessC!U\** 1/ DIII17SSSrNrr&c3.K|]}|VdSrIr])rmrnrls rLroz_size_of..s-55A}}Q''555555rNc34K|]\}}|VdSrIr])rmrIrnrls rLroz_size_of..s1@@DAq}}Q''@@@@@@rNzUnknown metadata type z on node rrrzO didn't have `val` metadata; we should always have `val` metadata on the nodes.)rrfrGr~)rrrrrsumritemsrrTrrrrrrrLr)rErrls @rL_size_ofrqsVTTTT   i c< ( ( &1dE] + + &555555555 5 T " " &@@@@CIIKK@@@@@ @ U\ * * & =%% %NDIINNNNOOO w* uy~/L/T T Tq eeee  rNc .ddlm}|t}|jD]'}|jdkr||jjxxdz cc<(tdt| tj dddS)Nr)rrr&%sTrsreverse) collectionsrr~rrrrXr=rrtrpr itemgetter)rrcntrEs rL _count_opsrys''''''%+c**C ++ 7o % %  $ % % % * % % %HHT6#))++8+>q+A+A4PPPQQQQQrN!list[torch._ops.OpOverloadPacket]cvg}ttjjD]}t tjj|}t |tjjsA|D]A}t ||}tj j |j vr| |nB|SrI) dirrrrrrrOpOverloadPacket overloadsr pointwiserr)r attr_nameopoverloadpacketoverload op_overloads rL pointwise_opsrs-/C((   "59>9==*EJ,GHH  (2244  H!"2H==Ky"k&666 +,,,7 JrNrtuple[Any, ...] depth_maprglist[tuple[fx.Node, int]]cfd|D}t|tjddS)Ncji|]/}t|tjjj&||0Sr])rrrrEr)rmargrs rLrzsort_depths..sE #z#ux}?Q/R/R Ys^rNr&Trt)rtrprrw)rr arg_depthss ` rL sort_depthsrsY'+J *""$$(*=a*@*@$ O O OOrNgmc4 tj i |jdD]} | fd |<dt |jjD d fd }ttt|jj}d }tj }|D]"}|j D]} ||kr |}|}#||St|jjd |D]:}|j d kr-|jtjjjjur ||;t|jj |d D] }||tj| }|S) a This pass finds the first bwd node in the graph (by looking at users of tangents) and then reorders the graph by walking from this node to all the way to the end of the graph. At each op in this traversal, we insert this op in a new graph and try to bring only the relevant subgraph from the other non-bwd edges relevant for this op. This closely mimics the behavior of autograd engine. Why is this pass required in the first place? This is an artifact of how partitioners work today. The starting point of partitioner is a joint graph, which is fwd and then bwd graph. In the case of checkpointing, we keep portions of fwd graph in their original place in the joint graph, while obtaining a bwd graph. As a result, the resulting bwd graph has copies of recomputed fwd subgraphs followed by the original bwd graph. If we run this naively, this leads to bad memory footprint, because the fwd subgraphs are live for way longer duration than necessary. This pass reorders the operations such that we prioritize the ops for the original bwd graph while only realizing those ops from the fwd graph that are necessary at any given point in the graph. rr'c|SrIr]rs rLrqz5reordering_to_mimic_autograd_engine..s ArNci|]\}}|| Sr]r]rmr;rEs rLrz7reordering_to_mimic_autograd_engine..s B B B93T3 B B BrNrErFrGr1cX|g}t}t|dkrO|}||vs|vr0||||jz }t|dkOt |fd}|D]}|fd|<dS)Nrc|SrIr])rnorders rLrqzSreordering_to_mimic_autograd_engine..insert_node_in_graph..+s %(rNrrc|SrIr]rs rLrqzSreordering_to_mimic_autograd_engine..insert_node_in_graph..-rrN)r$rrrall_input_nodesrtr)rE cur_nodesinsertable_nodesrrrs rLinsert_node_in_graphzAreordering_to_mimic_autograd_engine..insert_node_in_graphsF 0: )nnq  ==??D'''43;;   & & & - -I)nnq  ""28J8J8J8JKKK$ D DD!++D2B2B2B2BCCCII D DrNNrrErFrGr1)rrrr(rrrrrr6mathinfrrrrrrrr GraphModule) rrErrfirst_node_in_bwd minimum_ordertangentrnew_gmrrrs @@@rLrVrVs. I"$C##}#55@@''.>.>.>.>??D B B "(.(A(A B B BEDDDDDDDD&&bhn==>>NHM!))M ) )DT{]** %d $(! )   RX^$$%?u->'?%?@'' 7o % %$+9M9U*U*U  & & &RX^$$U+<%=%?%?@##T""""X ! !"i 0 0F MrNr_torch.fx.GraphModuler`fw_nodebw_nodedevice torch.device rng_countlast_fwd_inputlast_bwd_input#tuple[torch.fx.Node, torch.fx.Node]c|j}|td|j} |j} tjjj} |j|5|jd|} t|| j d<| }dddn #1swxYwY|j|5|jd|} t|| j d<| }dddn #1swxYwYt|j }| |d<|j|5| d| |jg|jR|}dddn #1swxYwY||| |t|j }| |d<| |5| d| |jg|jR|}||| |dddn #1swxYwY||fS) a% Note [CUDA Graph Safe RNG Functionalization] CUDA Graph capture doesn't work with get_rng_state and set_rng_state because these functions operate on CPU values, while CUDA Graph RNG capture uses on-device CUDA tensors. To solve this, we use graphsafe_set_state with a CUDA Generator registered to the CUDA Graph before capture begins. graphsafe_set_state updates the generator's pointer to reference a different GeneratorImpl, ensuring subsequent calls are correctly forwarded to the desired generator (and its cuda-tensor RNG state during graph capture). For each RNG operation's forward/backward pair: - We create two generators initialized with identical values - Each forward and backward call advances its respective generator equally - This keeps generators synchronized so forward and backward operations use matching RNG values When forward is called multiple times before backward (causing desynchronization): - We save the forward RNG state - We update the backward Generator's state before executing backward Before each CUDA Graph replay, replay_prologue updates captured RNG pointers with current states, ensuring backward Generator changes are reflected during replay. This function modifies both forward and backward computation graphs by: Creating RNG state placeholders for both passes Updating the forward node to use graph-safe RNG state Updating the backward node to use graph-safe RNG state For more details: https://github.com/pytorch/pytorch/issues/113541 Nzdevice_idx must not be Nonefwd_rng_state_rbwd_rng_state_ rng_staterrr)indexrrr_prims rng_primsgraphsafe_run_with_rng_staterKrr7rrr create_noderrrrinserting_before)r_r`rrrrrr device_idxfw_graphbw_graphr fwd_rng_state bwd_rng_state fw_kwargsfunctional_fw_node bwd_kwargs rng_outputs rL%apply_graphsafe_rng_functionalizationrKs^RJ:;;;HH#(<#9#V   ( ( 8 8''!334PY4P4PQQ $? $K$K 5!&'''''''''''''''  ( ( 8 8''!334PY4P4PQQ $? $K$K 5!& '''''''''''''''W^$$I*Ik  ( ( 1 1  %11  (.07<00 2                  !!"4555    gn%%J+J{  " "7 + +%%))  (.07<00 *  %%j111G$$$%%%%%%%%%%%%%%% > ))sI7BB!B?7DD D(E44E8;E8AH55H9<H9 num_sym_nodesc 6 'tj}dd}d d 'd!d }||}||}||} i} |jjD]} t | rtt | jd r_tjj | jj vrB| j |vs | j | vrV|| j } || j } | | j }| |d| | <tj j j}tj j j}d}|jdD]} d| j vr| }n|t#dg}t%t'|jd}t%t'|jd}t)'fd| D}|tjdt1|dk}tjj}t4jo| o|j p |jj}t?| D]V\}}|d} |d}'| }|j}|j}|r'|%|j dkrtC||| |||||\}}X|"| 5|#d|| jg| j$R| j%}|#dtLj'|dfi}|||j(d<|#dtLj'|dfi} tSj)| j(| _(| *| |+| |,|dddn #1swxYwY|"|5dt%|}!|-|!}"|||"j(d<dddn #1swxYwY|"|5|#d||"|jg|j$R|j%} |*| |+|dddn #1swxYwYX|rt%t]|jd}#|#j$d}$t1|$|z }%|$d|%t_|z|$|%dz}&|j0|&|j+|#|1|1||fS)"NgmodrrGrci}|jjD]I}|jdkr.get_rng_opssd+- J$ / /D?**DK00+I59III*. TY'rNrErFtorch.device | Nonecd|jvrdS|jd}t|ts|f}|D]5}t|tjr|jjdkr |jcS6tjdS)zV Check the example value of the node outputs to find the device type. rNcudacpu)rrrrrTrr)rE candidates candidates rL get_devicez)functionalize_rng_ops..get_devices  ! !4Yu% *e,, '$J# , ,I)U\22 ,#(F22$++++|E"""rNrrjc\ddlm}|}|td|5|H|jdkr=|t jcdddS|t jcdddS#1swxYwYdS)Nr)detect_fake_modezfake_mode must not be Noner) torch._guardsrrr from_tensorrr get_rng_state)rr fake_modes rLget_sample_rng_statez3functionalize_rng_ops..get_sample_rng_states2222222$$&&   !=>> >  @ @!fkV&;&; ,,UZ-E-E-G-GHH @ @ @ @ @ @ @ @(()<)>)>?? @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @s>B!/%B!!B%(B%r)fwdbwdrr'rzaCouldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see thisc3:K|]}|dVdS)rNr])rm node_pairrs rLroz(functionalize_rng_ops..sC)2 9U#$$rNrr&rrrrrrrrng_state_output_r)rrrGr)rErFrGr)rrrGrj)2r countrrrrrrrrrrrrrun_and_save_rng_staterun_with_rng_stater(rr)rr$valuesdiscardrrryrgraphsafe_rng_functionalizationfallback_random test_configs*graphsafe_rng_func_ignores_fallback_randomrrrrrrrrrrcopyrrrrr*rr recompile)(r r_r`ruidrrjoint_graph_rng_opsfw_graph_rng_opsbw_graph_rng_opsrecomputable_rng_ops_maprE base_noderrrun_and_save_rngrbw_tangent_start_nodefw_rng_state_outputsrrdevicesmulti_cuda_devices ind_config'use_rng_graphsafe_rng_functionalizationrrrrrrstater state_namebw_rng_state_nodefw_output_node fw_outputssym_node_start_idxrrs( @rLrUrUsf2 /  C    ####$ @ @ @ @&+l33"{9--"{9--!"(SS 4  S V,, S 1T[5EEE y 000DIEU4U4U+DI6I&ty1G&ty1G:A'2R2R $Y /|-D/B **m*<<  ! !$( ! E "$ o   +-(9?#=#=#=#O#OPPQQN(9?#=#=#=#O#OPPQQN6N6U6U6W6WG OOEL''(((W)'J. " " * * R&Q ,!**B*I*I*K*K L LP-P- 9E"E"G$$?? 4G -" v%%-R . . *NNN**733! 3! 3%-%9%9#$  #>&: & &"!,,#$,a0 - %9$8$@$@ 5!%11#$*2 #')GL"9"9 --j999##G,,,$++E222C! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3H**+@AA M M<c<< $,$8$8$D$D!0D0DV0L0L!&u- M M M M M M M M M M M M M M M **733 - -%11#&*! #>2   --j999##G,,, - - - - - - - - - - - - - - -& 3d9?#=#=#=#J#JKKLL#(+  __}< *** +()) *+,,- .  w'''"">222   i s8C'OO O (;P//P3 6P3 AR33R7 :R7 c|jjD]Y}t|jtjjr3|jjdkr#t|stj |j d<ZdS)z By default, the partitioner is not allowed to recompute collectives unless they come from a user-annotated AC region. See Note [Recomputing collectives in the partitioner] rrN) rrrrrrrrrr%rRr)r rEs rLrGrGst "(@@ t{EJ$9 : : @ %);;;"4((<&6%?DIk " @@rNcdfd |jjD]:}t|r)t|st |s |;dS)a\ Force save outputs from with_effects nodes wrapping effectful ops. Effectful ops (registered via _register_effectful_op) should not be recomputed because they may have arbitrary global side effects (I/O, RNG state, collectives, etc.). We mark the tensor outputs of with_effects as MUST_SAVE to prevent recomputation of the effectful op. The with_effects node returns a tuple (token, result). We recursively find all leaf outputs extracted via getitem and mark them as MUST_SAVE. Since these are saved, the with_effects op doesn't need to be recomputed in backward. rErFrGr1c|jD]h}|jtjurS|t |jdttfstj |jd<idS)Nrr) rrrrrrrrrr%rR)rErmark_getitem_outputss rLrz6force_save_effectful_ops..mark_getitem_outputssxJ H HD{h...$$T***!$)--"6"6 FFH-=-GDIk*  H HrNNr)rrrrr)r rErs @rLrHrHsHHHHHH"('' D ! ! '"4(( ').. ' !  & & & ''rNct}t|jjD]}|jdkr|jt jjj j u}|rmt|r | |j dt|r.|j d|vrtj|j djd<dSdS)Nrrr&r)r$rrrrrrrrrrrrrrr%rRr)r has_mutation_in_bwrEis_copy_s rLrIrIs5?LL+122 7h   ;%)."6">>  +D11 5"&&ty|444*400 LTYq\EW5W5W1A1K ! !+. EE!rNc|jtjkrdS|jd}t |t jurtdt |d|jvo |j dkS)NFrz#expected parent to be fx.Node, got rGr) rrrrrrrrrr)rEparents rLis_getitem_of_multi_outputrsk {h&&&u Yq\F F||27""Q4<<QQRRR  + J?0JJrNr+cX|jjD]}t|r|jD]S}t|rBd|jvr9d|jvr0|jd|jdkrt j|jd<T|jddr2td|jDst j|jd<d|jvrWtd|jDr9t|rd|j djvs|rt j|jd<|S)a If there are two consecutive checkpointed blocks with no operator in between, we would still want to stash the tensor at the boundary of checkpointed blocks. The following pass makes the last output node non-recomputable to allow for that. ac_graph_idrhas_backward_hookFc34K|]}t|VdSrIrr2s rLroz)cleanup_recompute_tags..sCEE)-t$$EEEEEErNc34K|]}t|VdSrIrr2s rLroz)cleanup_recompute_tags..s*@@TN4((@@@@@@rNr) rrrrrr%rRrrrr)r r+rErs rLrErEso"(+@+@ $  * @  H H"4((H%22%22 -049]3KKK-=-GDIk*y}}0%88 DEE15EEEBB D&*:)C +&  * *@@TZ@@@@@ ++400 +6CdiPQlFW5W5W$6X&6%?DIk " rNr]min_cut_optionsdont_ban)tuple[list[fx.Node], OrderedSet[fx.Node]]c <=>?@ABCDEFGHIJKttItrZtd|jD}|tdIjDz }t d|dbd =dbd >db=>Ifd A d dlGn"#t$r}td |d}~wwxYwdcAIfd CddCIfd }dcAfd BdeBIfd }Gj HtfstA| rd'} d}n#t j} d(}n|| j!\} }|rH| t jks | tDkr-H| jd z| jdz| d)|n(H| jd z| jdz| *| j#D]5}H| jdz|jd zt jd+6dhAfd0 }j$rj%D]}fd1|j#D}fd2|j#D}tM|d kr||tO|}tQ|j#D]}|rz)||kraA||rU|d?}ts|d@5}!|!:@dddn #1swxYwYn#tv$r}dA|dB}Yd}~nd}~wwxYwdC<Gj=j>?H?tod:dD?fdE=tH}"|"r i}#d6g}$didG}%|"D]!\}&}'}(|$A|'|&d6kr5|%|'})|#B|)gA|(W|'dkr5|%|&})|#B|)gA|(|%|&|%|'kr5|%|&})|#B|)gA|(|%|&}*|%|'}+|#B|+gAdH|*#g},|#CD];\}-}.|,AdI|-dJ|.D]}/|,AdK|/dS)r_overloadpacketN)rrrrr r&s rLroz solve_min_cut..sZ& & w/))gdkCT.U.U)  + , ,))))& & rNc34K|]}t|VdSrI)rrs rLroz solve_min_cut.."s94 4 CFF4 4 4 4 4 4 rNz&Ops banned from re-materialization: %sarFbrGr;c|jtjjjkrdS|jd}tjj|\}}|D]2}|j |}||urdSt|tr||vrdS3dSNFrT) rrrrauto_functionalizedr_higher_order_opsauto_functionalizeget_mutable_argsrrr)rr mutable_opmutable_arg_namesrIrrs rL!can_fuse_into_auto_functionalizedz8solve_min_cut..can_fuse_into_auto_functionalized's 8uy-A A A5VAY   # 6 G G     &  D(4.CCxxtt#t$$ 8844urNc|jtjjjkrdS|jd}|D]/}|jd}|t d||}||urdS0dS)NFtensors_to_clonerzkwargs must not be NoneT)rrrrr rr)rrrrrrs rL.can_fuse_into_triton_kernel_wrapper_functionalzEsolve_min_cut..can_fuse_into_triton_kernel_wrapper_functional;s| 8uy-N N N5H%78%  D(8,F~$%>???,CCxxtturNcHt|tjkrdS||rdS||rdS|jtjur*|jdjtjj j urdS |o |S)NTrF) r9rcatrrrrrrrr rM)rrrrop_typess rLrMz!solve_min_cut..is_fusibleHs 1   ) )4 , ,Q 2 2 4 9 9!Q ? ? 4 H( ( (q  y%FGG 5""1%%@(*=*=a*@*@@rNrzANeed networkx installed to perform smart recomputation heuristicsrEcz|rdSt|g}t|dkr|}|jD]P}|s||sdS|r||Qt|dkdSr)rTr$rrrryr)rErcurrrMr]rs rLis_materialized_backwardsz0solve_min_cut..is_materialized_backwardsbs   D ! ! 5v&& )nnq  --//C  ( ( //55 jjd>S>S 44##D))(MM$''' )nnq  urNrc@|jdkrdS|jtjurdS|jddt jkrdStj r |rdS|jtj j tjj fvrdSjr|sdSn?|rdS|rdSt'|rdSjr;|r0t*d |t/|jd S|jd kr|jtjkrd Sjr8t9d |jD}t=|}|dz|krdSdS)zRReturns reason string if node should be banned from recomputation, None otherwise.rNrzmarked MUST_SAVEznot in recomputable allowlistz random opzcompute intensive opznon-builtin opzmaterialized backwards: %s %szmaterialized in backwardiztoo far from backwardc3hK|]-}t|tjt|V.dSrI)rrrrqrs rLrozBsolve_min_cut..should_ban_recomputation..sM%% !*Q2H2H% %%%%%%rNrz reduction op)rrrrrrr%rRrrecompute_viewsrTrlift_fresh_copyr lift_freshrrVrRrPrrr=rrr dist_from_bwmax_dist_from_bwrrorrq)rEinput_tensors_size output_sizer!rrs rLshould_ban_recomputationz/solve_min_cut..should_ban_recomputationps 7o % %4 ;(* * *4 9==d + +/?/I I I%%  ! h&6&6t&<&< 4 ;4/79PQ Q Q4  2 (++D11 766 7!!$'' #"{,,T22 .--(.. (''  7 ..is_materialized..s/EE$zz$--EEEEEErN)rrr)rErMs`rLis_materializedz&solve_min_cut..is_materializedsA 7m # #4EEEEE$*EEEEEEErNrirbtuple[float, str | None]c tjr||vrdSt|}tjr#|rt jdfSt|jdtr.t|jdtj s tdfSt|dtt|jddzz}|r|dfS|d zdfS) zReturns (weight, cannot_save_reason). cannot_save_reason is None for finite weights, or a string explaining why the node cannot be saved for infinite weights. rzview op (recompute_views=True)rz$SymFloat (non-SymInt symbolic value)g?r~r&N)r treat_parameters_as_free_to_saverqr$rTrrrrrrrINT_INFr~rBrDr')rErimem_szr.rs rLget_node_weightz&solve_min_cut..get_node_weights  3 3337$  ! >h&6&6t&<&< >8== = di& 5 5 Gdi. == G FFF cST%6!.ban_recomputation_if_alloweds   D ! ! 5 8  4; (=>>@K)-?? > m u $   5 DI  *TYu-=u~"N"N 5   I X4:R0000@R    trNr_outsinkz;must be available for backward: input required for gradientr9r8z3must be computed in backward: required for gradientz+must recompute: marked by checkpoint policyz primal inputzforward RNG seedrrGznon-tensor outputz cannot save: )r:zdata dependency start_nodesr` max_ranger~cg}|D]-}tj| ||df.t|dkrtj|\}}}|s |S|jD]l} |rU ||kr1 || ||f}||vrtj||mt|dk|S)z Finds the first unfusible node in the chain of nodes starting from `start_nodes` and returns its position. Tr)heapqheappushrrheappoprry) rCrD sorted_nodesrnrIrEnode_is_fusiblerrrMr]s rLfind_first_unfusiblez+solve_min_cut..find_first_unfusibleh s4 9;  O OA N<)*@*@*C*CQ)M N N N N,!##',}\'B'B $At_" 4 --d333 : :++D11 : --d33i?? !..t44" 4..6C ,..|S999,!##rNcdg|],}||-Sr])ryrrmrr]s rLrz!solve_min_cut.. sK++D11&&t,,rNc>g|]}||Sr])ryrMs rLrz!solve_min_cut.. s<I4L4LT4R4RrNz1used above/below fusible %s:(%s) -> %s -> %s:(%s)r~ztoo long %s %s %s %srFTrrc dddS)Nmin_cut_failed_fx_graphrrr]r]rNrLrqzsolve_min_cut.. s5 (%%rNcSrIr]) fx_graph_strsrLrqzsolve_min_cut.. s<rNrmin_cut_failed_graphz.txtwz(failed to write: ) c dddS)Nmin_cut_failed_edge_listrrr]r]rNrLrqzsolve_min_cut.. s2$!!rNcSrIr]) edge_list_strsrLrqzsolve_min_cut.. s}rN node_namecndD]1}||r|dt| cS2|S)N)r8r@)endswithr)r[suffixs rL get_base_namez$solve_min_cut..get_base_name sO-99F ))&119(CKK<88889  rNz depends on z :z - z -> c dddS)Nmin_cut_failed_svgrrr]r]rNrLrqzsolve_min_cut..9 s 4$,))rNcSrIr]) svg_contentsrLrqzsolve_min_cut..= s{rNzFX graph dump: zMin-cut graph visualization: z[Production debugging: Use tlparse to extract debug artifacts (min_cut_failed_fx_graph, min_cut_failed_edge_list, min_cut_failed_svg)] aAOT Autograd failed to partition the joint forward-backward graph. The partitioner determines which intermediate values to save from the forward pass vs recompute in the backward pass. This error means a value is required for backward, but cannot be saved AND cannot be recomputed. This is a bug in PyTorch. Please file an issue at https://github.com/pytorch/pytorch/issues Nodes involved in the conflict: z [For PyTorch developers: one of the above constraints is wrong. Either the node should be recomputable, saveable, or not required for backward.] [Debug: min-cut path] z-Failed to compute min-cut on following graph:rschdjjSNrVjoin readwriteedgelistgenerate_edgelistnxr>srLrqzsolve_min_cut..d & ","7"I"I("S"STTrNchdjjSrfrgrlsrLrqzsolve_min_cut..n rnrNc3,K|]}||fVdSrIr])rmrnr>s rLroz solve_min_cut..v s,88Q $888888rNc3(K|] }|v|fV dSrIr])rmr non_reachableus rLroz solve_min_cut..w s1AAa=.@.@q!f.@.@.@.@AArNz node_in[:-3]=z != node_out[:-4]=ci|]\}}|| Sr]r]rs rLrz!solve_min_cut.. sHHHic4cHHHrNc3(K|] }|V dSrIr]rmrE name_to_nodes rLroz solve_min_cut.. s(22d 222222rNc|SrIr])rnode_idxs rLrqzsolve_min_cut.. s (1+rNrr)rrFrrFrGr;rW)rErFrGr)rErFrirbrGr/)r)rErFr6rrGr;)rCr`rDr~rGr~)r[rrGr)Kr$get_default_op_listr<rrDr=rnetworkx ImportErrorrDiGraphrrdrer;rrrrr5r4ryrrrrTrrCrrrrr1rir3rrrvrrBrrrrFrHrrG minimum_cutNetworkXUnboundedr;rhandlers owning_modulerrr_get_unique_pathopenwrite Exceptionrhrirjrk_find_infinite_capacity_pathr setdefaultrpvisualize_min_cut_graphrrrget_name_to_noderrt)Lrr]rrjoint_module_ops ops_ignoreder+r5r?rE ban_reasonis_non_tensor_nodeweightcannot_save_reasonrrK used_nodeordersfw_usersfirst_unfusible_usevisited start_nodefusible start_orderrIr  cut_value partition unbounded_excstructured_tracing_enabled fx_graph_filer finf_pathnode_constraintsraw_path_nodesr_ from_nodeto_noder6base from_baseto_baseconstraint_linesr[ constraintscconstraints_str raw_path_strsvg_pathlocal_files_msg tlparse_msg reachablecutsetnbrs cut_nodesnode_innode_outr0r=rrrZrRrMr.r!ryr{rrrmr>rrdrssL ``` @@@@@@@@@@@@@@@@rL solve_min_cutrs[ <<"$$H H%& & #)& & &    '4 4 $54 4 4 * *   9;GGG(    AAAAAAAA&  O           99999999vFFFFFF &$&$&$&$&$&$&$Prz||H(2 L@!YY 7h    9. . .9555!!I&!XX "!!I%!XP "  $       E!D      d   C ( (~ > > > >  & & C ( (/A B B B .-d33  # #D ) ) ;j ; ( (z : : :  " E}DI'EUty SDIe4Del)S)S%S  t   =..//F!%     e$$}6F&G 9%% 9%)""%8"")8i;** &F&  V6TX#5#579J9J    E! F";'9;;        di%/V1Cf  U U UJ  D    F" E!(      24,;"4 ; ;I%OF !*H6{{Q&:&:8S[[&Q&Q#!)/22;;D!0066;%22488;NNN&Jy$77O <//$O%%229==/ %22488 54T:::1#V'1||%+! V! VJ++J77 '' 33Z@2G$00<> C   **3//+2CCCG ))HH."!..s33!..z:: 10555IVVD!0066V&JsD11V! 44w1G1G1M1Mt0TUUU5g,,""8b-r~h&II 99 VVV%))*<%=%="%) #' "0  6  & ++!&tD,%%  0///     --CVLLMmS)) &Q %%% & & & & & & & & & & & & & & & 6 6 65555MMMMMM 6 ","7"I"I("S"STT  -,,,     099 ^ !68 &ZN ! ! ! ! /7  * 7F%%g...(((=11D$//b99@@HHHH&&(=33D$//b99@@HHHH"]9--w1G1GGG(=33D$//b99@@HHHH!. i 8 8I+mG44G$//<<CC1i11 +- *:*@*@*B*B : :& ; ''(9Y(9(9(9:::$::A$++LQLL9999:#ii(899O!;;~66L%nx.DiGraph[str, dict[str, Any]]!list[tuple[str, str, str]] | Nonectdg}tdgfg}|r|\}}||D]}||vr|||}|dd}|t jks |tkrW|dd}|||f} || gz} |dkr| cS||| || f|dS)zBFS from source to sink following only infinite-capacity edges. Returns a list of (from_node, to_node, reason) tuples representing the path, or None if no such path exists. rr:rr6unknownrAN) r$rpopleft successorsrrrr3rr) r>rqueuerE edge_pathneighbor edge_datar:r6new_edgenew_paths rLrr s ($$G>$   G '..I<<<<< = =1  '..I<<<<< = = . .' .9 . ..rNtuple[str | None, str | None]c ddl} ddl}n-#t$r tddYdSwxYw|j|}||d}| D]}|| | d}| t||tdkr|d |d }t%d d }t'|d 5} | |dddn #1swxYwY||fS)zVisualize the min-cut graph to an SVG file. Returns (path_to_svg, svg_content) tuple. Both are None if pydot is unavailable. rNzMInstall pydot to visualize the min-cut graph for debugging: pip install pydotT)exc_info)NNr:rredutf-8min_cut_failed.svgrT)r}pydotr~r=rnx_pydotto_pydot to_stringgraph_from_dot_data get_edges get_sourceget_destination set_labelrrC set_color create_svgdecoderrr) r>rmr dot_format dot_graphedgerrdrrs rLrr s   [    zz %%h//99;;J))*55a8I##%%""$//++,T-A-A-C-CDZP s6{{### U5\\ ! ! NN5 ! ! !&&((//88K 0&99H h     [  s &55FFFc .gtjtjtjtjtjtjtjtjtj tj tj tj tj tjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtj tj!tj"tj#tj$tj%tj&tj'tj(tj)tj*tj+tj,tj-tj.tj/tj0tj1tj2tj3tj4tj5tj6tj7tj8tj9tj:tj;tj<tj=tj>tj?tj@tjAtjBtjCtjDtjEtjFtjHtjItjJtjKtjL}tjItjJtjMg}|tjNtjOtjPtjRtjStjTtjUtjVtjWg z }|}|gtjtjXtjYtjLtjZtj[tj@tj[tj\tjRtjVtj]tjNtjStjOtj^tj_tj`tjatjbtjctjdtjetjftjgtjhtjitjTtjjtjktjltjmtjntjotjpz }|tjqtjrgz }||z }|tz }|tjtgz }|dtDz }t|}ttdtftjytjztj{g}tj|tj}tj~tjtjtjtjtjtjtjtjg }||z}t|t||t||S)Nc,g|]}t|Sr])r)rmms rLrz'get_default_op_list..l s! N N N1!3A!6!6 N N NrN.)rrrratan2rSrBrDpow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltrL bitwise_notceilfloorfracnegreluroundsilutruncr=log10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtrsqrtrRsigmoidsoftplus thresholdthreshold_backwardclampwherelerpaddcmulgelu gelu_backwardromean_grad_sum_to_size sum_to_sizerMtotype_asrrsqueeze unsqueezersub_to_copyaliasrslicetrNbroadcast_in_dimexpand as_stridedpermuteselectrvrOclone full_likevarstd _unsafe_viewrbroadcast_tensors scalar_tensorones new_zerosr%arangetriuvar_meanisinfrfullzerosempty empty_likeargmaxmaximumiota'_low_memory_max_pool_offsets_to_indicesrgatherr zeros_likerr$rr native_dropout rand_like randn_likemm convolutionconvolution_backwardbmmaddmm#_scaled_dot_product_flash_attention'_scaled_dot_product_efficient_attention_flash_attention_forward_efficient_attention_forwardupsample_bilinear2d _scaled_mmr?)default_recomputable_opsrecomputable_view_opsrCrDrBrAr@s rLr|r| ssL: L: L: L: L:  L:  L: L: L: L: L: L: L: L: L: L: !L:" #L:$ %L:& 'L:( )L:* +L:, -L:. /L:0 1L:2 3L:4 5L:6 7L:8 9L:: ;L:< =L:> ?L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:X YL:Z [L:\ ]L:^ _L:` aL:b cL:d eL:f gL:h iL:j kL:l mL:n oL:p qL:r sL:t uL:v wL:x yL:z {L:| }L:~ L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:Z"\4>4:F           %H$! $! "$! $! $!  $!  $!  $! $! $! $! $! $! $! $! $! !$!" #$!$ %$!& '$!( )$!* +$!, -$!. /$!0 1$!2 3$!4 5$!6 7$!8 9$!: ;$!< =$!> ?$!@ A$!B C$!D  E$!F 5G$!$LT[ 99( / ! N N N N NN!":;;HS#X./ dndo>J   !   0 4 % )    #Z/K ())8   rNc2i}|jD] }|||j< |SrI)rr)rryrEs rLrr s-')L ''"& TY rNmemory list[float]runtimes max_memoryall_recomputable_banned_nodes"tuple[float, list[int], list[int]]c0tj}|dkrt|||S|dkrt|||S|dkrt |||S|dkrt |||S|dkrkt dtj ||||}t ||t| t| St|tr||||||\}} d || fStd |) Ngreedyilpdpr)dynamic_memory_budget_dpzdynamic_memory_budget_dp is an experimental solver. It does not guarantee performance improvements. Additionally, it is not guaranteed to be stable.)rrW recorded_knapsack_input_memories recorded_knapsack_input_runtimes)graph_info_provider) knapsack_algomax_mem_budgetrBz,Not aware of memory budget knapsack solver: )ractivation_memory_budget_solverr*r+r(r)r=warningr'inialize_from_graphr,get_knee_point_memory_budgetrrr) rrSrUrVr]rWSOLVERr`saved_node_idxrecomp_node_idxs rL#_optimize_runtime_with_given_memoryrj sr 3F vx<<< 5FHj999 468Z888 3 3 3-fh KKK - - -  ?   0C#*G-3-5        $7   **))+    F0 1 1T*0& KY8U+ + '^_55R&RRSSSrN no_dispatchrrjct|j}d fd fd|D}fd|D}|||S) Ndtorch.SymInt | intrGr~c&t|S)Nri)r )rnrjs rLrealize_symbolz8_remove_symbols_without_guarding..realize_symbol s X6666rNc&g|] }|Sr]r]rmrrqs rLrz4_remove_symbols_without_guarding.. s# . . .1^^A   . . .rNc&g|] }|Sr]r]rss rLrz4_remove_symbols_without_guarding.. s# 4 4 4AnnQ 4 4 4rN)stride)rnrorGr~)rshaperunew_empty_strided)rrjrvrurqs ` @rL _remove_symbols_without_guardingrx s MME777777 / . . . . . .E 4 4 4 4 4 4 4F  uV  4 44rNc tj}dd}|dkrdS|dkrnt5dd lm}t j|jjf\ |  fd }|cdddS#1swxYwYdS|d krdd l m }t j|jjf\ |d 5}j i dddn #1swxYwY| }t|dSt|t r |St#d|)Nrr rGcvt|tjrAt|jdtjrt |jddSt|tjrAt|jdtjrt|jddSt|tjr't|jdtj rdSt|tjr't|jdtj rdS|S)Nrrhri?T) rrrrrrTrxrr rrrks rLmaterialize_argz)estimate_runtime..materialize_arg s a ! ! j &M&M 3AF5MDQQQ Q 27 # #  16%=%,(O(O $QVE]TBBB B 27 # #  16%=%.(Q(Q 3 27 # #  16%=%-(P(P 4HrNtestingr&profiler) benchmarkercjiSrI)r)rrrEsrLrqz"estimate_runtime.. s;4;3O3O3OrNflops)FlopCounterModeF)displayz Not aware of runtime estimator: )rr rGr )r*activation_memory_budget_runtime_estimatorrl$torch._inductor.runtime.benchmarkingrrtree_maprr benchmark_gputorch.utils.flop_counterrrget_total_flopsrBrrr) rE RUNTIME_MODEr|rmsrmode counted_flopsrrs ` @@rLestimate_runtimer sDL    y  q  " " ]]   H H H H H H!??TY t_d?}|0|tb2d@|!|AdS)HNr&rzJThe valid ranges for memory budget are 0 <= m <= 1. The provided value is )rrrrrF)rrrrr0r`rGrCcLttt|dz SNeA)romaprq)r0s rLestimate_activations_sizez:choose_saved_values_set..estimate_activations_size" s3x..//#55rNszc|dz z z Srr])r max_act_size min_act_sizes rLget_normalized_sizez4choose_saved_values_set..get_normalized_size+ sS\L899rN activationsc,|z z z SrIr])rrrrs rL get_mem_ratioz.choose_saved_values_set..get_mem_ratio. s())+66E < '  rN)rrr)r)get_node_storagec3.K|]}|VdSrIr])rmrErs rLroz*choose_saved_values_set..L s/TT4 0 0 6 6TTTTTTrNr=rbc"fd|DS)Nc|g|]8}|jtdkr|vst|6|9S)r)r'r~r)rmr<rinput_storagess rLrzRchoose_saved_values_set..get_recomputable_banned_nodes..Q s`    S))$$Q''~==033>  >==rNr])r=rrs rLget_recomputable_banned_nodesz>choose_saved_values_set..get_recomputable_banned_nodesN s3      !    rNcdg|]-}|jddtjk+|.S)rF)rrr%rRrs rLrz+choose_saved_values_set.._ sA 6::k5 ) )-=-G G G G G GrNcg|]}|v| Sr]r])rmr<must_save_nodess rLrz+choose_saved_values_set..d s*!!!0H0H0H0H0HrNTrtc@g|]}t|Sr]rq)rmr<rs rLrz+choose_saved_values_set..q s8-.HQKK((rNc,g|]}t|Sr])rr&s rLrz+choose_saved_values_set..t s.#'rNrkrr]r_rrtuple[list[fx.Node], float]c  5t| t|d| \}}}dddn #1swxYwYt}|D].} | |#t$rY+wxYw| st dt|| |\}} tr"t| |||d D | ||fS)Nrz:dont_ban must be a subset of all_recomputable_banned_nodesc,g|]}t|Sr]rrs rLrzNchoose_saved_values_set..get_saved_values_knapsack.. s+'''$%HQKK'''rN) rrWsaved_node_idxsrecomputable_node_idxsexpected_runtimememories_banned_nodes normalized_memories_banned_nodesruntimes_banned_nodesmin_cut_saved_values) rjrBr$r BaseExceptionissubsetrrr<r )rr]rrrrrr;r0rIaggressive_optionsrWrrlrs rLget_saved_values_knapsackz:choose_saved_values_set..get_saved_values_knapsacky s[]]   4%%M1%%-    &               )3 )  C  :3?@@@@       !>??  L (        a !  4'.K /'=!1'')F'''2G&;%1    ---s!'?AAA99 BBrtuple[float, float, float]cb|\}}|t|z |fS)N)r]r)ro)rr0rrrrr]rs rLestimate_for_budgetz4choose_saved_values_set..estimate_for_budget sU-F-FYK... *L*)**-== l++ rNrBr{gMbP?r1cg|] }|d S)r1r]rmitems rLrz+choose_saved_values_set.. 000DG000rNcg|] }|d Sr&r]rs rLrz+choose_saved_values_set.. rrN) )figsizeo)markerz.4fz offset points)rrcenter) textcoordsxytexthaz Memory Budgetz Runtime of Recomputed Componentsz:Pareto Frontier of Memory Budget vs. Recomputation Runtime)exist_okr_rank_memory_budget_paretorIrz%Generated Pareto frontier curve at %s)rr]r)r0r`rGrC)rrCrGrC)rr`rGrC)r=rbrGr`)rrCr]r_rrrGr)rrCrGr)3rrrban_recompute_used_far_apart!ban_recompute_long_fusible_chains#ban_recompute_materialized_backwardban_recompute_not_in_allowlistban_recompute_reductionsaggressive_recomputationrrartorch._inductor.fx_utilsrr$rtrqrtorch.utils._mode_utilsrlvisualize_memory_budget_paretorrsortmatplotlib.pyplotpyplotfigureplotrannotatexlabelylabeltitlegridgcfshowrgetcwdmemory_budget_pareto_dirmakedirsrrris_initializedget_rankrrhr2savefigr=rd))rr]rrruntime_optimized_saved_valuesrImore_aggressive_optionsmore_aggressive_saved_values%aggressive_recomputation_saved_valuesr=rrecomputable_banned_nodesroptionsbisectslhsrhsmidpltx_valuesy_valuesr<txtfigfig_dir rank_suffixfig_namerrWrrrrrrrrrrrlrs)`` @@@@@@@@@@@@@@rLchoose_saved_values_setr s qMA-- hYf h h   $$A#)#K%+%O & E8 O& ! "'',).$)    (5))%"A --6666-,Y-=>>L,,-KLLL|##--:::::::        &##(%*  '4Y 7''# !}122]BB++  %;HY 2;;7)<}:;;mKK44999999TTTT9CSTTTTTN       !> =l K K*O !!!!,!!!%+!x%%%! ())Q../112O+H4333331.1.1.1.1.1.1.1.1.1.f,AG          '&s++-@-@-E-EF 1:abb>WQZ^ + + GAJ/0G /";;==Sq6CF?T))NN3'''NN3'''))3q6CF?a*?@@qrr7c!""g%%NNC:...qrr7c!""g%%NNC:... /  ''''''0000000000 7 ### 8C000 ))  FAs LL hqk"*      ?### 5666 NOOO ggii  )++  * 65G K$ / / / /   ) ) + + B0A0P0P0R0R BA5#4#=#=#?#?AAK7<< TKTT:L:N:NTTT   H ;XFFF % $#yk     rNlist[torch.fx.Node]cXddlmdd}dfd }tjrwtjrXtjd kr5||r)||rt55d |Dg}d ttjD}tj ||dt|g}i}t|D]t\}} fd | D} d} | D]B} t| } | | z } |tj kr | || j<C| |d <|| utj|tjj}tj|tjjjjt-tj|}d|d|t3ddfdfd||D}dddn #1swxYwYdddn #1swxYwY|S)Nr)unset_fake_temporarilyrr@rGr;c|jD]7}t|jtjjr|jjdvrdS8dS)N>c10d_functionalrTF)rrrrrrr)rrEs rLhas_collectivesz3_sync_decision_cross_ranks..has_collectives sP%  D UZ2 +'+RRRtturNc0dd|jD}tj|d}dt tj Dt55tj |dddn #1swxYwYdddn #1swxYwYtfdDS)N/c3$K|] }|jV dSrIrrmrs rLrozE_sync_decision_cross_ranks..has_same_nodes.. s$>>qAF>>>>>>rNrcg|]}dSrIr]rmrIs rLrzF_sync_decision_cross_ranks..has_same_nodes.. sNNNqdNNNrNc30K|]}d|kVdSrr])rmr all_inputss rLrozE_sync_decision_cross_ranks..has_same_nodes.. s,::!:a=A%::::::rN) rhrhashlibsha256encode hexdigestr:rrget_world_sizerlall_gather_objectr)rnode_strrarrs @rLhas_same_nodesz2_sync_decision_cross_ranks..has_same_nodes s 88>>K,=>>>>> 8 899CCEENNE%*;*J*J*L*L$M$MNNN ]] D D2244 D D   / / F C C C D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D::::z::::::s6# C2.!C C2C C2"C #C22C69C6r&cg|] }|j Sr]rrs rLrz._sync_decision_cross_ranks.. s5551555rNcg|]}gSr]r]rs rLrz._sync_decision_cross_ranks.. s%::::::rNc g|] }| Sr]r])rmop_namerys rLrz._sync_decision_cross_ranks..' sTTT|G4TTTrNz total size)rr'zpicked_rank_idx=z, saved_nodes of current rank=rc dddS)N)aot_joint_graph_sync_decision_cross_ranksrrr]r]rNrLrqz,_sync_decision_cross_ranks..= sG (%%rNcSrIr])sync_decision_cross_ranks_strsrLrqz,_sync_decision_cross_ranks..A s#@rNrc g|] }| Sr]r])rmrnrys rLrz._sync_decision_cross_ranks..D s*$% QrN)rr@rGr;)torch._subclasses.fake_tensorrrrrrr rlr:rrrrqrrrridistributed_c10d_get_object_coll_device all_reduceReduceOpMAXr~argminrr)rr0rrobjectssaved_ops_names_all_ranks saved_sizessaved_ops_with_sizesr;saved_ops_names saved_nodes saved_sizerE size_of_nodesaved_sizes_tensorpicked_rank_idxryrrs @@@rLrTrT sEDDDDD ; ; ; ; ; ; &&((1   , , . .1   , , . . 2 2 OK ( ( 3 N; ' ' 3]]* * 2244* * 55 5556G::!%"3"B"B"D"DEE::: %   / /0I7ST: V V V+K88L%'K35 (12K(L(L / /$_TTTTOTTT  'GGD#+D>>L,.Je/88:::::F,TY75?$\2"":....!&(9QQSS"""    ( ("u'8'I'R'V )   "%,/A"B"B"G"G"I"IJJO-E-E-EoC-E-E ) A@@@     )B?)SLQ* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * X s7 J&GJ< JJ JJ JJ#&J#modulercrd}|rdnd}tt|jd}|jdtjjjD]}t||j dj }t|tj rg}|jdD]}||jvr|j|5|j|d|} |d z }|jd | jd <| }|| d d d n #1swxYwY|r|j|5|jdtjjjg|j |Ri} || d d d d n #1swxYwY|jd} | r"| \} } g| d|DR}|| f| jd<|j||S)u Graph-safe RNG lets torch.compile use CUDA Graphs for graphs with RNG ops. For graphs without HOPs, the partitioner adds placeholder nodes fwd_rng_state_* and bw_rng_state_* to the forward and backward graphs. At runtime, the AOTDispatcher retrieves these RNG states and passes them to the compiled graphs. This works well for no-HOP graphs. With HOPs, the partitioner runs recursively: it first partitions the HOP (producing forward/backward HOP subgraphs) and then stitches them back into the outer joint graph. For HOPs that contain RNG ops, the outer joint graph now includes HOP subgraph modules with extra RNG placeholders. We must thread these placeholders through the outer module partitioned forward and backward graphs—this function does exactly that. It collects the RNG placeholder nodes from the HOPs and creates corresponding placeholders in the outer forward and backward graphs. There is a catch: for a short period, the joint graph is in a “bad” state. The HOP subgraphs expect additional inputs (because of the new placeholders), but the outer graph call sites don't yet provide them. We can't fix this in the joint graph because the joint graph's input signature is fixed (primals, tangents). As a compromise, we keep the joint graph in somewhat of a bad state for some time and, once the outer forward and backward graphs are partitioned, insert the corresponding RNG placeholders and wire up the calls. rrrrr'r)rrrIr&rNT)propagate_metaeager_input_valsc(g|]}|jdS)r)r)rminps rLrz2thread_graphsafe_rng_from_hops.. sDDDc#(5/DDDrN)r)rrr(rrrinvoke_subgraphrrrrrrrrKrrrrrrr)r+rr rng_string last_inputhop_nodernew_rng_inputsplaceholder_nodernew_hop_node_with_fixed_args eager_vals eager_args eager_kwargsnew_eager_argss rLrXrXK s<I$/D_Jhv|66-6HHIIJJJL++ 59#9#I,,2,268=#3#:;; h / /( 2,.N$,N$=$=$=$O$O 9 9 !1!666 55jAA99$*L$<$<)77I77%% "Q 0@0Ee0L u-%. &--i888999999999999999 2\11(;;  39<3K3K' .>9(-9.99 440 224T3               &]../ABB  /9,J &#&DD^DDD&&N '$M056HI ''111 Ms&.AE  E E 3AGG G list[int]c t|jt|jjD]n}|jdkrd|jvr|n$t|r||vr|j ottt|jj}ttt|jj}||z}t||\}}} } } d|Dt!|j||| d} tfd| jDtfd|jjD} tfdt#|D}d }i}|jjD]}|vr |||<|d z }t%|| | ||S) Nrtangentsrc34K|]}||jdk|VdS)Nrr')rmrs rLroz!classify_nodes.. s;!-ADH4D4D4D4D4D4DrNrc3HK|]}|jdk |jVdSr(r)rxs rLroz!classify_nodes.. sC88  7h   TY    88rNc3,K|]}|v|v |VdSrIr])rmrErdrvs rLroz!classify_nodes.. sF66  ( ( (T9J-J-J -J-J-J-J66rNc3*K|] \}}|v |VdSrIr])rmr<pr#s rLroz!classify_nodes.. s;--a!7T2T2T2T2T2T2T--rNrr&)rrr$rrrrrrrrrr5r4r/rrrr_)r r#r!rErrrar+r,r-r.reforward_only_graphrfrifw_cntrhryrdrvs ` @@@rLrJrJ s $L$677L-7\\"(11 7m # # dk(A(A  ! !$ ' ' ' ' !$ ' ' (  ! !$ ' ' ' $ $ $  $ $TZ 0 0 0 L,>,DEEFFM!&)P>V"W"WXX 3 3F OOOCK/1B)--//<FK1BI.88888&,888.. ,666666 &,666,,O #----- ..---##FH"( $ $ $#HTN aKF #  rNr)r#compilerc |j||j}tjrt |}||_|j}t |}t|} |rt|d}tj st|t|t||g}t|||} t| jdkrt!||||| jSt%|jjD]} | jdkrt+d| _"| | sd| _?t+d| _| jD]$} t3| j| jdz| _%tj} |jD]?} t7| jd dt<r| jd } n@t?|| | }tj rtA||}d dtCtEfd|}tCtEtF|}tCtEd|}tI|||||| j\}}|r$| r"tK|||t|\}}tM|}tj'rddl(m'}||||| jtS|}tS|}tU|d}tU|d}tVrtYd|D}t[d|Ddz }t\/d|t\/d|tad|jjD}tad|jjD}||z}tct*}|jjD]G} | j2|vr._is_assert_only_symbool..Gs3WW!AH  = EEWWWWWWrN)rrrrrrrrrns rL_is_assert_only_symboolzDmin_cut_rematerialization_partition.._is_assert_only_symboolCs\ qvzz%((%- 8 8 XAG q  XWWqwWWWWW rNc:t|o | SrIr<)rnrKs rLrqz5min_cut_rematerialization_partition..Ls!k!nnG-D-DQ-G-G)GrNcBt| ot| SrI)rr1rJs rLrqz5min_cut_rematerialization_partition..Qs[^^+EN14E4E0ErNr=r?rATcJg|] }t|t|f!Sr])rqrrs rLrz7min_cut_rematerialization_partition..|s)KKK SVV4KKKrNc34K|]}t|VdSrIrrs rLroz6min_cut_rematerialization_partition..s('J'J 'J'J'J'J'J'JrNz'Theoretical Activations Stored: %.2f GBz,Theoretical Per Activation Storage Sizes: %sc3:K|]}|jdk |jVdSrNr)r&s rLroz6min_cut_rematerialization_partition..9% % 47o;U;UDI;U;U;U;U% % rNc3:K|]}|jdk |jVdSrQr)r&s rLroz6min_cut_rematerialization_partition..rRrNr z# remat/fw/bw: %d/%d/%drtzCount of Ops Rematerialized: %sr):rrrrcser8rrrErFrGrHrIrJrrdrarirrrr~r'ryrrDactivation_memory_budgetrrrrCrrTrrr1r"rUrVr@rWr:rXr<rtror=rr$rrrrrr rprrw)r r$rFr!r#r cse_graphrr[r\r]rErrr0rrr_r`r@ sorted_sizestotal_activations_size_gbfw_module_nodesbw_module_nodes remat_nodescountsrematerialized_opsrKs @rLrDrD sD**,,,  Dz' && & $K!5l!C!C%=l%K%K"!X-lQVWWW  :-|,,,\***|,,,$,(*%3_I 9 &''1,,  +*G(1(M     +122RR 7h   #CD  ))$// R !D   #CD   R R$'(94;Lq;P$Q$Q!! R3M! dimmOT::E B B  Io6M E +#L (M1+|LL      G G G G  O f^\BBCCEE|TTL 4'-'$-$IIy" ) #8iC4H4H$$ Iy4I>>I*         %$     1    y))Iy))I.yeLLLI.ydKKKIHKKlKKKLL %('J'J\'J'J'J$J$JS$P! : >DyK''GDKAR,S,S's4;677888A=888  %         $ LLNN 3A 6 6    24FGGG i rNfx_graphTtracedfnamefigname clear_metaprogstr | list[str] | Noneparse_stack_tracedot_graph_shapec|rDtj|j}tj||}|jjD] }i|_ tj |\} } | sdtj z} t d| | tj||||} | } t#| d| dz} | | }| | |dS| ||dS)NruzWriting FX graph to file: %s%s)rerfwrite_)rc)rdeepcopyrrrrrrrsplitextrtorch_compile_graph_formatr=rr# FxGraphDrawerget_main_dot_graphrlstrip)r_r`rarbrcrerfrrErextgr write_methods rL draw_graphrrs'M&,//  22L&  DDII  ''ID# 6F55HH -tS999"+'    A A1hC899LNSNNE | U U&&&&&&rNrW)rrrGr;)rErFrGr~)rrFrGr)rErFrGr)rErFrrrGr)NF)rrrar`rr`rrrrrr;rGr)r rr!r~rGr")r0r`rrrGr1)r6r7rGr~)r>r?r) rr@rErArBrCrDrCrEr~rGrA)rr@rErAr\rAr^r_rQrCr`rCrEr~rGrA)rirjrGrC)rGrp)rErArGr;)rGr_)rwr_rGr)r)rr@r!r~rGr1)rr@rGr1) rrrrrrr!r~rGr1)Nr) r0r`rrrrrirr!r~rGr1rI)r rr0r`rr`rrr!r~rirrr;rr;rGr) r rr$r r!r~r#r%rirrGr)rlr~rwr_rGr~)rrrGr1)rGrz)rrrrgrGr)rrrGr)r_rr`rrrArrArrrr~rrArrArGr) r rr_rr`rrr~rGr)r rrGr1)r rr+r;rGr) rrr]r_rrrrrGr )r>rrGr)rrrrrGr)r>rrGr)rGr?)rrrGr)rrrSrTrUrTrVrCr]r_rWr`rGrX)rrjrjr~rGrj)rErFrGrCr)rrr]r_rrCrGr`)rr@r0rrGr)r+rrr;rGr)r rr#r<r!r~rGr_)r) r rr$r rFrr!r~r#r%rGr)r^TNFN)r_rr`rrarrbr;rcrdrer;rfrrGr1) __future__rrrr rFr loggingrrros.pathrrBrvrrcollections.abcr dataclassesrrtypingr r rtorch._inductor.inductor_primstorch.distributedtorch.fxrtorch.utils._pytreeutils_pytreertorch._dynamo.utilsr r ;torch._functorch._activation_checkpointing.ac_logging_utilsr $torch._functorch._aot_autograd.utilsrtorch._inductorrr!torch._inductor.custom_graph_passrr"torch._library.fake_class_registryrtorch._library.utilsrtorch._loggingrrtorch._logging._internalrrr%torch.fx.experimental._backward_stater"torch.fx.experimental.proxy_tensorrrtorch.fx.experimental.sym_noderr%torch.fx.experimental.symbolic_shapesrrrr r!r"torch.fx.passesr#torch.utils._ordered_setr$torch.utils.checkpointr%r-_activation_checkpointing.graph_info_providerr'"_activation_checkpointing.knapsackr(r)r*r+,_activation_checkpointing.knapsack_evaluatorr,_aot_autograd.descriptorsr-r.r/_aot_autograd.functional_utilsr0_aot_autograd.graph_compiler1_aot_autograd.logging_utilsr2_aot_autograd.utilsr3r4r5r6r7 compile_utilsr8r9r:r}rmsympydebug_partitionerr<r\ getLoggerrXr=rrrNr?r_rrrrrrrrrrrrrr rrrrrrr/r5r=r]rhror{rrrrrrrr"rar~r3rdrqrycacherrrVrrUrGrHrIrrErrrrr|rrjrrlrxrrrTrXrJrDrrr]rNrLrs """""""     ********$$$$$$********%%%%%%%% %%%%$$$$$$$$$<<<<<<<<A@@@@@555555@?????++++++77777777......AAAAAA??????HHHHHHHHLLLLLLLL)(((((//////333333LLLLLL LKKKKK A@@@@@777777;;;;;;IHHHHHHHHHLLL%66666'g'111111 y~  >>>>>>> >2          B       IIIIo 2 - - - -H $) rrrrrjXXXXCCCCBBBBJJJJKKKK    5555JJJJ$ KKKKK\C!C!C!C!L9999"GGGG45555    FGFGFGFGFGRMGMGMGMGh XXXXX~?C ' ' ' ' ' \04 k"?C$)&+k"k"k"k"k"k"f7;>B b b b b b b J #c(("""":RRRR "PPPPKKKK\[*[*[*[*|a a a a H @ @ @ @''''<.KKKK5555x,0 v &v &v &v &v &rB / / / /$!$!$!$!NeeeeP-T-T-T-T`0/////5555*N*N*N*N`z z z z z zNNNNbOOOOd>>>>H| 7; | | | | | | D#'#"&'''''''rN