x-j5ddlZddlmZddlZddlmZddlmZm Z m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZddlmZddlmZddlmZmZdd l m!Z!d d l"m#Z#d d l$m%Z%m&Z&d d l'm(Z(ej)j*Z*ej)+Z,gdZ-gdZ.eej/Z0ej1j2j3Z4dZ5dZ6e&dGdde%Z7dZ8dZ9e*j:fdZ;dZdZ?dZ@ej1j2jAe4fdZBdZCdZDd ZEd!ZF d-d"ZGd#ZHd$ZId.d&ZJd'ZKd(ZLGd)d*ZMGd+d,ZNdS)/N)reduce) ParallelModeis_data_parallel_reduce_opis_parameter_related)new_process_group) _get_comm_group get_logger get_var_numelinsert_dependencies_for_varsis_backward_opis_dep_skip_op is_forward_opis_optimize_op*naive_set_dist_op_attr_for_program_by_mesh6naive_set_dist_op_attr_for_program_by_mesh_and_mappingset_var_dist_attr get_var_size)core)default_main_programdefault_startup_program) unique_name)_is_master_grad_cast_op)PassBase register_pass)AutoParallelStreamType)create_py_readercreate_double_buffer_readerreadslicesplitassignsend_v2) adamadamaxadamwdecayed_adagradmomentum dgc_momentum lars_momentummerged_momentumlambsgdfloat16cn|jdod|jdvS)N op_namescopez/auto_parallel/reshard)deschas_attrattrops p/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/passes/auto_parallel_sharding.py_is_reshard_opr8Qs< 7     C "bgll>&B&B BCauto_parallel_shardingceZdZfdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZdZdZdZdZdZdZxZS) ShardingPasscHt|dd|dd|dd|dd|dd|dd|dd|dd|d d|d d|d d|d g|d d|dd|ddt|_g|_i|_d|_d|_g|_ dS)N dist_contextstagesharding_degreedegreeenable_overlapparam_comm_stream_numgrad_comm_stream_numparam_bucket_size_numelgrad_bucket_size_numelpartition_algorenable_hierarchical_comm params_grads global_rank amp_dtyper/gradient_sync_after_accumulateF) super__init__set_attrset dp_groupssharding_infosvarname_to_sharding_infosharding_hybrid_dpouter_dp_groupshared_params_grads)self __class__s r7rOzShardingPass.__init__]s  nd+++ gt$$$ '... h%%% &--- -t444 ,d333 /666 .555 '... 0$777 nb))) mR((( k9--- 6>>> (*%"'"#%   r9c|ddS|ddvrdS|dDt|dtr|ddkrdSn[|dDt|dtr|ddkrdSndSt|dd krdSt|d tr|d d krdS|d dS|d dS|d dS|ddS|ddS|ddS|ddSdS)Nr>Fr?)rr@rrArIrrJrBrCrDrErFrGrHT)get_attr isinstanceintlenrXs r7 _check_selfzShardingPass._check_selfus  == ( ( 05 == ! ! 2 25 ==* + + 7t}}->??EE 011Q66u7 ]]8 $ $ 0t}}X66<< BBBBuB 5 t}}^,, - - 2 254==77== $-- C C CC5 ==) * * 25 ==0 1 1 95 ==/ 0 0 85 ==2 3 3 ;5 ==1 2 2 :5 ==* + + 35 ==3 4 4 <5tr9cdS)NT)rX other_passs r7_check_conflictzShardingPass._check_conflictstr9c|d|_t|dp|d|_t|d|_t|d|_|d|_t|d|_t|d|_|d |_ |jd ks |jd kr|js Jd t|d |_ t|d |_ |d|_ |d}| | }}|d|_|jdkrtjjj}d}||||jD]C} || ||| || |D|d|j|||dS)Nr>r@rAr?rJrBrCrDrHrz3multiple comm stream need enable_overlap to be TruerErFrGrIrLbfloat16)r] _dist_contextr_sharding_world_sizer?rJrBrCrDrHrErFrG global_blockrLrVarDescVarTypeBF16_build_sharding_groupsblocks_shard_optimizer_shard_gradient_synchronization_shard_parameterrPrW_optimization_pass) rX main_programstartup_programcontextrI main_block startup_block__amp_target_dtype____amp_target_dtype_name__blocks r7_apply_single_implzShardingPass._apply_single_impls!]]>::#& MM+ , , G h0G0G$ $  w//00 t}}];;<<"mm,<==%(7N)O)O%P%P"$' 6L(M(M$N$N!(, &) ) %  % ) )T-F-J-J&  E  &(+ MM3 4 4( ( $'* MM2 3 3' ' # $}}->??}}^44  % % ' '  ( ( * *" {33 >Z ' '#'<#7#< (2 % ##J ===!( 8 8E  ! !% 7 7 7  0 0 7 7 7  ! !% 7 7 7 7)ABBB  o>>>>>r9c\|||||dSN) _collective_data_parallel_groups_build_sharding_infos)rXrxrIs r7roz#ShardingPass._build_sharding_groupss2 --j999 "":|<<<<>>>|(@||jrjy|||?>>?T%==BBB|(@||jrjy|||CBB#x~555s$"2ssbjbpsss655|$$(@@@@A\):):AA_c_wAAAA@@ !999*.'1A55550144444>**a///K0//2MND$N22."3>!B!B&7&G&G##!)1?D  .( $ M   & &} 5 5 5&- J J> FYz~%>%> ? ?@@7 ?7 ?GC*:r::5 ?tzA~~ 1!4 ()@;+;+;C+@+@)@A 88DDD))#E):::**;U*CCCOOO "$'--"4"466J!+,Bjooc.B.B,B!CJ88DD6"))*555#?G%%c:666G&&uj9999w"<<<"$'')"4"4#%#6q#9",/(";"--c->>>"::!0%*G$4 ' ' ' +W # ;   !.OO '" O&N3/%2%2 .%.%7 #--c->>>!!#####r9c||jdkrdSgd}t}t}tt|jD]\}}t |s|j|vr|dd}|d|d}| |s=| ||jdvr|j D]} | | ttt|jD]0\}}t |s||vr| |d1|D]} || dtt|jD]\}}t |s|jd krg} |jD]}||vr| ||jd| |j d} t|jD]\} }||| zd zd d | gid | gid|jjdddt.jjjt6t8ji}|j| | }n|!dS)Nr[)elementwise_mulsquared_l2_norm clip_by_normrr@GRAD)rrFrsumr all_reducexoutring_idr1z /gradient_clip_model_parallelism reduce_typerinputsrr)"r?rQrrr_is_gradient_clip_oprrrrrrrrrinput_arg_namesrr2rrS _insert_opridpaddle distributedReduceOpSUMrOpRoleOptimizerirvarr)rXrxremoved_op_typeremoved_op_idxremoved_tmp_varrr6rrrvarname reserved_vars sum_op_outputirnew_oprs r7_shard_gradient_clipz!ShardingPass._shard_gradient_clipks :>> FQPP%%Ijn5566 = =GC'++ w/))XXc]]1- '(B*//'*B*B(BC 88DD="&&s+++w"EEE+-+>==K+// <<<<Yz~%>%> ? ?@@ 7 7GC'++ n$$%%c%666& 8 8G  " "7 " 7 7 7 7Ijn5566! ! GC'++ w% " "$"499J!88%,,Z888!!#}555 " 3A 6 (1$2E(F(F$A}'22a! ) #m_5!& 8%}':'=*,N)6+=+F+J'  3  F*KK&NN=99I= @ !!#####r9c|jdkrdSttt|jD]#\}}t |st d|dS)Nr[z$weight decay is NOT supported by now)r?rrrr_is_weight_decay_oprr)rXrxrr6s r7rz ShardingPass._shard_weight_decays :>> FYz~%>%> ? ?@@  GC&r** ): !!#####r9c<g}ttt|jD]\}}t |sn|jt vrd|jvsJt| ddksJ| dd| s>| fd|j D| |d|j|ttt|jD]C\}}t|j dkr&|j d|vr| |dD|D]Z}||r||d||r||d[||dS)NParamrrc g|] }|k| Srdrd).0rrs r7 z@ShardingPass._shard_optimizer_ops_and_states..s. '&*44$444r9Fr)rrrrrr_supported_optimizer_type input_namesr`rrextendrrrWr_get_param_gradhas_varrr)rXrxryshould_removed_optimizer_statesrr6rrs @r7rz,ShardingPass._shard_optimizer_ops_and_statessU*,'Yz~%>%> ? ?@@  GC!"%% w333".0000288G,,--2222XXg..q1 88DD 3::+-+>))#E)::::,33,,Z88 Y}/@%A%A B BCC : :GCB'((A--'*.MMM((5(9996 ? ?G!!'** <&&wU&;;;$$W-- ?))')>>>!!###$$&&&&&r9c t|jdks |jdkrdS|jD]}|jD]}||jsJ||jsJ|dd|id|id|jjd| |jttj i}| d d tjz|j|}|Jt%||j|j|j|j |dS) Nr[r broadcastrrrrootrr1/r)r?rErSrrr append_oprr get_var_rankrrr _set_attrr DataParallelrirrrrrr)rXrxryrrrparam_dist_attrs r7rz)ShardingPass._insert_optimizer_broadcastssx :>>T9A== F!0  M&-  !))%*55555$,,UZ88888#--$<"EN!=#6#9 : :5: F F#V_ .    "C,*C$C&GGNN '222F#0#0&,5 + 8 !!#####r9c\||jvsJ|j|}||Sr)rTis_in_local_shard)rXrrs r7rz)ShardingPass._is_parameter_in_local_shards7T:::::5jA ..z:::r9ch||jvsJ|j|}||}|J|Sr)rTget_param_grad)rXrrp_gs r7rzShardingPass._get_param_grad sDT:::::5jA **:66 r9c |jdkrdSd|jD}ttt |jD]\}}t ||rh|jdkr(|dtj j ks3|jdkr<|dtj j krd}tj j }nd}tj j }|j d}t|}|j|} t!||||| jj| ||j|} |jr| |s||dzd nC|d |jj|d d t4jzt9||rU|jd} t| }|j|} | |s||d |dS) Nr[cg|] }|j Srdrrrs r7rz@ShardingPass._shard_gradient_synchronization..<<> F<%> ? ?@@/ ;/ ;GC*2z::# G|++ ..$-2CCCGx'' ..$-2CCC%-N"&-"3KK%-N"&-"3K/2 9*EE $ =i H -"!'*!..y99&   / (::9EE ))#')>>>>LLD,?,BCCCLL&l.G(G%R44 ;.q19(CC $ =i H $66yAA;))#E):::!!#####r9c |jdkrdSd|jD}|jD]y}||\}}g}|D]A}||dkr3|||jkr||Bttt|j D]\} } t| r| j D]} t|| |jr+|| s|| D| |vrI|| } | |jkr| } nt!j| dz} || }|| |j|jd}|j|}t1|j||j|j|j|j| }|| }| | | || |tA|| | |j| |j!j"| #d|jttt|j D]]\} } | j$d kr| j d} | j%d}| |vr.|&| d |'|d ^ttt|j D]'\} } tQ| j%d ksJ| j%d}| j$d kr| #d |vr^|j)r?|||jkr!| *d |j)j"nM|&| d n5|||jkr|&| d | j$d kr9||vr5|||jkr|&| d )|D]N}|||jkr.|'|d |'|d O{|+|+dS)Nr\cg|] }|j Srdr r s r7rz1ShardingPass._shard_parameter..Or r9rz @BroadCastF)rrr persistablerrcastrrrr),r?rRrS"get_broadcast_vars_and_param_usager local_rankrrrrrrr_is_param_fp16_cast_op param_namesrrgenerater create_varrrrirrrrrget_op_dist_attr_for_programget_input_dist_attr _rename_inputset_input_dist_attr_insert_init_and_broadcast_oprrr4rrrrr`rVrr)rXrxryrrneed_broadcast_vars param_usagenot_used_param_namerrr6r root_rankbroadcast_varname input_varnew_var ref_dist_attr op_dist_attrinput_dist_attrrs r7rszShardingPass._shard_parameterKs :>> F<>$/00(..z:::#D:>)B)B$C$CDD@ @ R!"%%"$"4<<J."B (A! $@@LLC/66zBBB !)<<< - : :: F FI M$<<<,6)),7,@&5--)%/NN:$>$> ","7"7!2"+/"+/(- #8##!.OO )& * .#)6)6%2%; !.KKBOO%+7*J*J&++((5FGGG$88-2")%0!%+. ***    g<|$D:>)B)B$C$CDD D DR7f$$/2  1!4 !444))#E):::**;U*CCC#D=3D)E)E$F$FGG > >R2.//14444 1!4 7k))wwy))[88 /F - : :; G G,7!8!8LLD4G4JKKKK)44Su4EEEE*66{CC,788*44Su4EEEG{**#{22%22;??$/00",,Su,===) F F !..z::$/00**:E*BBB!--ju-EEE  F !!###$$&&&&&r9c|jdkrdSd|_d|_d|_t |jdks Jdt |jd|jd}t j||5| ||j dkrA|jdkr| |n |jd kr| |ddddS#1swxYwYdS) Nrsharding_coalesce_grad_sharding_coalesce_param_rKzZgradient synchronization optimization only support one sharding group right now, but got [z].rr[r\) r?grad_coalesce_prefixparam_coalesce_prefixcomm_op_scheduling_priorityr`rSrstatic program_guard_gradient_sync_optimizationrE&_fuse_overlap_parameter_comm_stage_two(_fuse_overlap_parameter_comm_stage_three)rXrurvrs r7rtzShardingPass._optimization_passs :?? F$=!%?"+-(4&''1,,, FilmqnAjBjB F F F-,,+A. ] ( ( G G Q Q  , ,] ; ; ;+a//:???? NNNNZ1__AA-PPP Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q QsA"C77C;>C;c|jdkr |jsdSt}t }|||\}}|||||dS)Nr)rFrBrrkr _group_grads_overlap_grad_comm)rXrrxrycoalesce_to_group_mapgrad_name_to_group_maps r7r7z(ShardingPass._gradient_sync_optimizations  &! + +T5H + F)++88:: /11>>@@ 8<8I8I  9 9 55    ! "      r9cT t}t}t||j\}}t dt d|jdt|dt|di}|j rg|_ |j j }t|jD]M}|dkr|j } nt|d} |j | t"jjd Nt d |jd i|_t+|D]7\}} t| d ksJt| d krt-j|jt3|z} || | jdd || | jdd | _|dd| ji| j| jdddddd| jt>t@j!in| jd| _t d|dtEd| jDdt #d|dd| jDd| || jj$<||jz} |j | d} |j | d}|dd| jid | jid!| j%d"| j&t>t@j'i}| |j|<|(d#d$tRj*z|j r||j+_,|j-|j+_.9i}t+|j/D]\}}ta|rz|1d%d}|j|}||} d}|j r |j+j,}t||jkr6te|}|j|1d&d}nT|j/||jz }ta|s Jd'|j|1d%d}||g|g|fg||<t| jd kr]| j3r.| jd(}||||g|g|f|||d z|g| j|fti|d)}|D]j}||ddd(D]V\}}}}tk|||||j6t@j'd(gd*d*d+, }|j r|||j+_,|j-|j+_.Wk|7dS)-NzSharding Stage2 Optimization:zParam Bucket size is [], [z] Parameters are fused into [z ] BucketsrTforce_new_group comm_group comm_streamz#Parameter Communication would use [z ] streams.rrrr stop_gradientcoalesce_tensorInputOutput FusedOutput copy_data use_alignrrzBucket[z] size [c,g|]}t|Srdrrps r7rzGShardingPass._fuse_overlap_parameter_comm_stage_two..Es)T)T)Ta,q//)T)T)Tr9z]MB.z] parameters: cg|] }|j SrdrrPs r7rzGShardingPass._fuse_overlap_parameter_comm_stage_two..Hs+M+M+MqAF+M+M+Mr9.rDrErrrrrr1rrParamOutz:Unexpected: sharding broadcast pre op should be broadcast.rKreverseFsharding_stage2_broadcast_depr is_recomputerr1)8rrkr group_paramrE_loggerinfor`keysrBparam_comm_group_stream_pairsrrrangerCrrrSHARDING_STREAMrop_to_stream_idxrrrr3strrr coalesce_varrrrrForwardrdebugrrrankrrrrrexecution_streamr4scheduling_priorityris_sharding_param_broadcast_opoutput_get_broadcast_first_depend_oprsortedr rir)rXrrxrygroup_to_param_mapparam_to_group_mapbroadcast_var_to_group_maprrr param_groupcoalesce_var_namecomm_stream_idxrDrErdep_mapr6r) broadcast_var prior_varpre_op last_gradindicer prior_vars post_vars depend_ops r7r8z3ShardingPass._fuse_overlap_parameter_comm_stage_twos@)++88:: /11>>@@ 1< 472 2 ..  4555  nT%A n nsK]KbKbKdKdGeGe n nEHI[I`I`IbIbEcEc n n n   &("   ' 24D .!'-E4566  66)/EE-eTJJJE299&+'='M'S LL\d6P\\\   %'D !'(:(?(?(A(ABBE E NA{{##q((((;!##$/$8.Q7%%!((*%+ $"& ) ,6+@+@*%+ $"& ,A,, ( ''*#[%56"-"2'2'? $T#T!2#V^ (    ,7+;A+> ( LL[![[S)T)T;CS)T)T)T%U%U[[[    MMP!PP+M+MK d t> d |jdt | dt |di}i}i}i}g}tC|D]r\} }t |jdkr|"tGj$|j%tM| z|j'dd |_(|j)*|jd}tW|j)|j(|j,|j-|j.|||j<|j/}|||<|0|j|jr8|j/}|||<|0|jn|jd|_(|jD] }|||j1< |||j(j1<tt |2}t |2}t |} |3|}!t |!dksJ|3| }!t |!dksJ|3| }!t |!dksJtitktC|jD]\}"}|"|vr||"}|jd} | |jdj1ks!Jd|jdj1d| d|6| |j(j1|7| |j(j1|"|vr|8|"d|"|vrb||"}|jdj1}#|#|jvsJd|#d|dd|jD}$g}%g}&|jD]@}'|'j9}(|%0|(|&t |(A|:|"dd|$i|$|j(ddddd d |j'd!|%d"|&tvtxj=i#})|j)*|j(}t}|)|j-|j,|j)|j.t||"||j|j(|j)txj=dgddd$% }*|@||fS)&a conditions for gradients to be grouped: 1. group size < grad_bucket_size_numel 2. same dp group (TODO) 3. same src rank 4. same dtype 5. dependency: grad would NOT be used by other ops within group segment main logic: 1. record coalesce group 2. record all dp allreduce/reduce op idx 3. insert coalesce op 4. insert coalesce dependency (avoid allocate memory too early) 5. modify and remove allreduce/reduce op 6. ensure sharding-dp hybrid parallel logic gradients inside same group would be fuse into one coalesce tensor rNrct|j|jz}d|jD}t ||dkS)Nch|] }|j SrdrSrrs r7 zHShardingPass._group_grads..op_depend_on_group..s888c888r9r)rQrrrr` intersection)r6rvars_ var_namess r7op_depend_on_groupz5ShardingPass._group_grads..op_depend_on_groupsP*R-@@AAE88UZ888Iu)))44559 9r9rrzZSharding should reduce grad first and than allreduce if Hybrid Sharding with Data-Parallelr[Trz@Hybrid Sharding with Data-Parallel should sync same gradient varz-Sharding Gradient Communication Optimization:zGradient Bucket size is [r@z] Gradients are fused into [z ] Buckets.FrFrrKz$Unexpected: it is supposed to sync [z ] but got [rrz-Unexpected: op is supposed to generate grad [cg|] }|j SrdrS)rgrads r7rz-ShardingPass._group_grads..ts???Ddi???r9rHrIrJrMrNrconcated_shapesconcated_ranksrsharding_grad_coalesce_deprY)ArrFr rVarGrouprQr`rrr4r rrrrrr acceptablecollectrrcoalesce_op_idxr coalesce_dep_varnamerreduce_op_indicesrUrrrallreduce_op_indicesr\r]rrrrr2rcrrdrirrrrrpoprrr^rrrr"_rename_outputrrrrrBackwardrr r)+rXr|rrfirst_backward_opr6first_backward_varname cur_group grad_groupsgrouped_grad_namesrr is_reduce grad_namerrggrad_varjdep_op dep_varnamer>r=modify_reduce_op_mapcoalesce_op_mapremove_reduce_op_indicesrr,last_reduce_op_idxlast_allreduce_op_idxrcoalesce_op_set modify_op_set remove_op_setconflictrfirst_grad_name grad_namesrrgrad_r coalesce_opr|s+ r7r;zShardingPass._group_gradss 0i  & * * +,D '   Bb!! $&!   $ F!2!CA!FT899   UU : : : #c((llQB)"--6 BGx/BGGM4J4JM%M%O5 !py/2 :9EE $11*== 99Y//''$776$,>>>>>%%h5555&&y111 ()D E EI%%h555y~&&!++01AI-A(QU44Q)QU44 QZF"("9!"?,33E4NOOO%*Z]"  : :49&ty11=B !%"4"9 : :o2244550557788 455 "// >>8}}!!!!"// >>8}}!!!! --m<<8}}!!!! Yuy%9%9 : :;;H H GC***,S1/2  EJrN$7777g5:b>;Ngg[dggg877  E,>,CDDD!!)U-?-DEEE...  5 111o%%',"'*Q-"4&"*====eOee`beee>==@?EJ??? "$!#"Z66E!KE#**5111"))#e**5555#;;*#Z0",','9 $U#T)?(.#V_ < &&GG* G!.!.&*3 9IIe899&&O""'!=     $&<<.|s4   /)-AAAAAAr9z:Sharding Gradient Hierarchical Communication Optimization.zcurrent global rank idx: rTzlocal inter node ranks idx: c&g|] }|zk |Srdrd)rrgnode_idxrs r7rz3ShardingPass._overlap_grad_comm..s3   ?*h66666r9zlocal intra node ranks idx: root_idrrrr1r)7rBgrad_comm_group_stream_pairsrrr`rDrrrrrrr4rrrrrkrdrr`rrrUrrhr4rirrr rr]rmr^r rirrrrrrHrjrJr\r]rrr_rrrrr)/rXr|rr=r>rrrstreamrrtreduce_op_countgrad_comm_op_to_stream_idxrr6 stream_idxrDrEreduce_varname grad_grouppost_idxnext_opryrzr{r1rrr| global_group global_ranksinter_node_ranksintra_node_ranksinter_node_groupsintra_node_groups_grad_comm_stream_idxinter_node_groupintra_node_groupdst_rankin_peerintra_node_dstinter_node_dstrrrrs/ @@@r7r<zShardingPass._overlap_grad_comms &"  F,.)#)t011  AAvv%+)%FFF544F  - 4 4"'#)    i%'" ~~g g GC)"--d %7l**rww}/E/E&/3J00,t/HH 1;*2.!>zJ  #? K! "$5!1!1!!42>B !.3~EEEEz''!++  &OB/&3'4L  $GCL #QwH.&:3O& A CL''$&3&O'4L    1< -4 0 Y 666*z/K!#'lG"<<77GLL%==*37=== #>>%003~EEEE:EG%68%91HCGx'' ..$-2CCC&8D MM*JKKD'))') CL''!G&3&3 5L     1$ 1HCC 555( ( ADDbD!' ' *-FFFEED09&O""'!-4S (>!%.%; .!*!3  #.?J +< <"+?K' T  (Z  O+o=BBBB(.L'-L#'#3o#E '?:H     (     LLL    LLHT5EHHH I I I LLK8HKKK L L L$%%+>????     (     '((O;;;; LLK8HKKK L L L "  " 4455  !((%&6MMM"((%&6MMM $D59)=)=$>$>??/ / R-b11.8++GGM22dm6GGGGH,Fb+I('89M'N$'89M'N$%'YYu%5%5a%8N*D&(l&C "2779#5#566H#G/15III"&%-%?NLL,<,?@@@LLN;;;)1_)D!&!>!>!G!)$'#8 %~%!*+;+> )> -t}/@ +V_ #"? " "((*C,2K,K .@KF,= $ @#,@ r9)__name__ __module__ __qualname__rOrbrfr}rorrrqrrrrrrrrrrsrtr7r8r9r;r< __classcell__)rYs@r7r<r<[s&&&&&0&&&P3?3?3?j===*/J/J/Jb E E E=$=$=$~B$B$B$H $ $ $&'&'&'P!$!$!$F;;; 6$6$6$pE'E'E'NQQQ2   "o$o$o$b   |=|=|=|```````r9r<cX|jD]}|jtvr|cStd)NzCould not find optimizer op.)rrr Exception)r|r6s r7rlrls@i 7/ / /III 0 2 3 33r9c ||}||} ||dd|id|id|d|t|i} | ddt jzt| | j| j || j ||krZ||d d |j id |j d |j t|i} t| | j| j || j dSdS)z% empty op for initialization rrrrrrr1rremptyrrrrN)rrrrrrrrrrrrrr) r| insert_idxrrr(rrr>rubroadcast_var_dist_attrrs r7r$r$sWIIg&&M*KK * * W~ w I  +  F ^S<+D%DEEE:,,(1 Y.. M./,,W /    ?  # 0 # 0 ,5       r9c b|dks Jd||||d|gid|gid|d|d|t|i} |||} t | | j| j|| j | d d tj z| S) Nrz5root id should be a positive int, but now root id is rrrrrrrr1r) rrrrrrrrrrr) r|op_typer reduce_varrrr>rrrrs r7rrs a<<<III << * * j\" % w w ;   +  F== *I;#  ^S<+D%DEEE Mr9cd}d}t||z|g}t||||}t||||}||fSNrr)r`r) origin_groupsharding_group_sizergdp_axis sharding_axisrrrs r7rr;sVGM   "5 57J KE|UGTBBH$\5-NNN ^ ##r9c|jdo,|jddS)Nr1z/gradient_clipr2r3r4 startswithr5s r7rrFsB 7  N + + # 11j!""#r9c|jdo,|jddS)Nr1z/regularizationrr5s r7rrLsB 7  N + + $ 11j"##$r9cXt|sdSt||ttjjjsdSt||rdS|jd}|d| d}| |sdS| |j S)NFrr) r _is_desired_cast_oprzrrlrmFP32rrrrr is_parameter)r|r6rrs r7rrRs "  u  r')=)B  uub))u%a(K3k..s3334I == # #u 99Y   ,,r9crt|rdSt||sdS|jd}||vrdSdS)NFrT)rrr)r|r6rrs r7rrbsPbu ub ) )u#A&Ju 4r9c8|jdkrdSt|jdksJt|jdksJ||jd}||jd}|j|ks |j|krdSdS)NrFrrT)rr`rrrr)r|r6 src_var_type dst_var_typer* output_vars r7rrms  w&u r! " "a ' ' ' ' r" # #q ( ( ( ( ",Q/00I2.q122J,&&**:l*J*Ju 4r9cd}d|vr|d|d}nCd|vr|d|d}n!d|vr|d|d}|S)Nz.cast_fp16@GRADz.cast_bf16@GRADr)r)rrs r7rrsII%%A /@ A AAB i ' 'A /@ A AAB I  7 w 7 778 r9ct|sdS|jd}t|}||sdS||jS)NFr)rrrrrrr6r|rrs r7r r s_ %b ) )u%a(K-k::I == # #u 99Y   ,,r9ct|sdS|jdkrdS|jd}t|}||sdS||jS)NFrr)r rrrrrrrs r7rrsq "  u w%u%a(K-k::I == # #u 99Y   ,,r9c|jdko?|jdo%tj|jdvS)Nrr1)rr2r3rrr4r5s r7rjrjsI ; F G  ^ , , F  %n)E)E Er9cd}|jD]}t||j|s||}|j}||}|j} t|dkrd|d} | dkrA| | dkr5|| |kr-t|j |j| |} t| }n|S)NrrKr) rrr|r rget_input_dims_mappingrr`r process_idsr) rank_idr6r>rrrrrinput_dim_mapping mesh_shapebatch_size_axis group_rankss r7rrsH( $J,GG $AA"EEI$1L ) @ @ L L %+J$%%**/2O## ?(Ca(G(G?o&@&@"1$0$*' ##K 1==HE Or9c*i}d}g}|D]-}t|}||z }|||f.dt|D}d}d}|D]<\}}||dz|dzz|z kr|dz }|||||z }=|S)z shard the continuous param into same rank and divide the forward&backward computation into segment, which will favor the fuse pass in later. we assume that the params is already sorted by utilization order. gci|]}|gSrdrd)rrs r7 z*partition_by_use_order..s000q"000r9rg?r)rrr`) r group_sizemappingtotal_param_mem param2memrmemcur_rankmem_accus r7partition_by_use_orderr sGOI''5!!3%&&&&00eJ//000GHH s o+x!|z*partition_by_greedy_even..s AEr9rzparam [z#] should larger than 0, but it is [r)r`indexminrrrr)rrrrank_sizesrrgnumels r7partition_by_greedy_evenrsGz"" C* E{{3u::&& U###))5;::qyyy Mej M MU M M Myy d u Nr9 greedy_evenc ||dkrt||}nt||}td|D]e\}}td|dt d|Ddtdd|Dd f|S) NrzSharding Parameter Partition:zRank:z, Parameter Size:c,g|]}t|Srdrrs r7rz(partition_parameters..s ,L,L,L3\#->->,L,L,Lr9z MB.zParams in this rank: cg|] }|j SrdrSrs r7rz(partition_parameters..s-D-D-D3ch-D-D-Dr9rT)rr r\r]itemsr)rralgorrank_to_paramskvs r7partition_parametersrs 1&*EE/ CC LL0111$$&&II1 SA S S,L,L!,L,L,L(M(M S S S     G-D-D!-D-D-DGGGHHHH r9ci|D]\}}||f|j<g}|jD]K}|jD]}|vr||vr|| t |t krnL|jd}i} t |j} g} t |r|jtvrxttt|jD]n\} }|jtvr[t | ddksJ|| | dd<| | ot |t | ksJ|D]l} | d}|j | | j |||| | m| D]} || d|t |j| ksJt&d |d fd |DS) NrKrrrnop)rFrz(Sharding the Order of param being used: rTc g|] }| Srdrd)rrQpname_to_pg_pairss r7rz$re_order_program..=s 4 4 4Q a 4 4 4r9)rrrrr`rrrrrrrrr2 copy_fromset_op_dist_attr_for_programr rrr\r])r| param_gradsr>rQg use_orderr6rlast_op pname_to_opnum_opsremove_op_indicesrpnamerr!s @r7rr s++1%&F!&!!Ii, - -J///)++  ,,, y>>S!233 3 3 E 4imGK%)nnGg)7<3L#L#LYuy%9%9 : :;; . .GCw333288G,,--222246 BHHW--a01!((---9~~[!1!11111  E__%_00F K ! !+e"4"9 : : :  5 599+e:LMM     % . .C   Su  - - - - 59~~(((( LLHIHHHIII 4 4 4 4) 4 4 44r9ci}i}g}t|}|jD]}||j}|||r|||n%t|}|||||j|_||vr!|||jn |jg||<|||j<||fS)zA param are group by: rank id fuse_size dtype )rrrrrrrr)r fuse_sizernrobucketrrrgs r7r[r[@s F##I%33))%*55   t , , +   eT * * * * ++I   eT * * *&3&E&E J' ' # * * * y ) 0 0 < < < <-2ZL y ))25:&& 1 11r9c2eZdZdZdZdZdZdZdZdS)rc||_d|D|_t|jtt|jks Jdd|D|_d|jD|_|j|_||_|j |j|_ ||_ t|j|j|j |_i|_|dS)Nc(i|]\}}|j||fSrdrS)rrQr%s r7rz)ShardingInfo.__init__..es$EEE1QVaVEEEr9z&found duplicated param in params_gradscg|]\}}|Srdrd)rrQrs r7rz)ShardingInfo.__init__..js222TQq222r9cg|] }|j SrdrSrPs r7rz)ShardingInfo.__init__..ks888qAF888r9)rrIr`rQrrrrrJrrrrGrr param_to_rank_map_param_to_rank)rXrrgrIrGs r7rOzShardingInfo.__init__cs EE EEE4$%%S1B-C-C)D)DDDD 4EDD32\222 88DK888,+++D,<==.2 K$*>    !!!!!r9cl|jD]\}}|D]}||j|j<dS)z@ mapping parameters to the rank which holds it. N)rrr4r)rXrgrrs r7r5zShardingInfo._map_param_to_rankxsV!/5577 6 6LD& 6 615"5:.. 6 6 6r9c2||jvr |j|SdS)NrK)r4)rXrs r7rzShardingInfo.get_var_ranks# d( ( (%g. .rr9c>|||jkSr)rrrXrs r7rzShardingInfo.is_in_local_shards  ,,??r9ct}t}i}t|jd}|jD]5}t |r|jD]}||jvr||xxdz cc<6|jD]}t|||js|jd}|jd}| || ||||<||xxdzcc<|j ||j |<| D] \} } | dkr| | !||fSr) rQdictfromkeysrrrrrrrr4r) rXr|broadcast_vars fp16_params fp16_to_fp32r&r6rrrusages r7rz/ShardingInfo.get_broadcast_vars_and_param_usagesee  mmD$4a88 ) 1 1Bb!!  0 1 1 !111 +++q0+++ 1) M MB)%T5EFF +A.J-a0K   { + + + OOK ( ( ((2L %  # # #q ( # # #.2.@.LD { + +'--// * *LE5qyy""5))){**r9c||std|d||jvrtd|d|j|dS)Nzparam[z] not in current rank.z] not in params_grads)r ValueErrorrIgetr9s r7rzShardingInfo.get_param_gradst%%j11 JHjHHHII I T. . .GjGGGHH H $$Z666r9N) rrrrOr5rrrrrdr9r7rrbsq"""*666 @@@+++:77777r9rc&eZdZdZdZdZdZdS)rc||_d|_d|_d|_g|_d|_d|_d|_g|_g|_ d|_ dS)NrKrF) max_sizerrgrrrdrrrrr)rXrFs r7rOzVarGroup.__init__s[       $(!#!#$&!!&r9c|jdkrdS|j|jkrdS||jkrdS|jt|z|jkrdSdS)NrTF)rrrgr rFrXrrgs r7rzVarGroup.acceptables^ :??4{dj((uty  uzM%0004=@@u4r9c|j|_||_|xjt|z c_|j|dSr)rrgrr rrrHs r7rzVarGroup.collectsF[   mE***  r9c*t|jSr)r`rras r7__len__zVarGroup.__len__s49~~r9N)rrrrOrrrKrdr9r7rrsP ' ' '      r9rr)r)Ologging functoolsrrpaddle.distributedrr 8paddle.distributed.auto_parallel.static.operators.commonrrr5paddle.distributed.auto_parallel.static.process_groupr-paddle.distributed.auto_parallel.static.utilsrr r r r r rrrrr7paddle.distributed.fleet.meta_optimizers.sharding.utilsrpaddle.frameworkr paddle.staticrr paddle.utilsrauto_parallel_master_gradr pass_baserr pass_utilsrop_proto_and_checker_makerrkOpRoleAttrNamerrrINFOr\rlrmFP16rzr{r8r<rlr$rrrrrrrrrrr rrjrr rrrr[rrrdr9r7r]s !!!!!!                           QPPPPP!!!!!!GGGGGGGG$$$$$$>>>>>>.............. ( /-==??         *W\ " "|+0%CCC'((rrrrr8rr)(rj+4446 6 6 D O&&&&R$$$### $$$ - - - %*% & - - - - - -(,>4(     151515h222DM7M7M7M7M7M7M7M7`!!!!!!!!!!r9