Αi5SSKrSSKJr SSKrSSKJr SSKJrJ r J r SSK J r SSK JrJrJrJrJrJrJrJrJrJrJr SSKJr SSKJr SSKJrJr SS K J!r! S S K"J#r# S S K$J%r%J&r& S S K'J(r( \RRRTr*\RRRW5r,/SQr-/SQr.\"\R^5r0\RbRdRfr4Sr5Sr6\&"S5"SS\%55r7Sr8Sr9\*Rt4Sjr;SrSr?Sr@\RbRdR\44SjrBSrCSrDS rES!rFS,S"jrGS#rHS$rIS-S%jrJS&rKS'rL"S(S)5rM"S*S+5rNg).N)reduce) ParallelModeis_data_parallel_reduce_opis_parameter_related)new_process_group) _get_comm_group get_logger get_var_numelinsert_dependencies_for_varsis_backward_opis_dep_skip_op is_forward_opis_optimize_op*naive_set_dist_op_attr_for_program_by_mesh6naive_set_dist_op_attr_for_program_by_mesh_and_mappingset_var_dist_attr) get_var_size)core)default_main_programdefault_startup_program) unique_name)_is_master_grad_cast_op)PassBase register_pass)AutoParallelStreamType)create_py_readercreate_double_buffer_readerreadslicesplitassignsend_v2) adamadamaxadamwdecayed_adagradmomentum dgc_momentum lars_momentummerged_momentumlambsgdfloat16cURRS5=(a SURRS5;$)N op_namescopez/auto_parallel/reshard)deschas_attrattrops p/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/passes/auto_parallel_sharding.py_is_reshard_opr7Qs8 77    C "bggll>&B BCauto_parallel_shardingc^\rSrSrU4SjrSrSrSrSrSr Sr S r S r S r S rS rSrSrSrSrSrSrSrSrSrSrSrSrU=r$) ShardingPass[c>[TU]5 URSS5 URSS5 URSS5 URSS5 URSS5 URSS5 URSS5 URSS5 URS S5 URS S5 URS S5 URS /5 URS S5 URSS5 URSS5 [5Ul/Ul0UlSUlSUl/Ul g)N dist_contextstagesharding_degreedegreeenable_overlapparam_comm_stream_numgrad_comm_stream_numparam_bucket_size_numelgrad_bucket_size_numelpartition_algorenable_hierarchical_comm params_grads global_rank amp_dtyper.gradient_sync_after_accumulateF) super__init__set_attrset dp_groupssharding_infosvarname_to_sharding_infosharding_hybrid_dpouter_dp_groupshared_params_grads)self __class__s r6rOShardingPass.__init__]s  nd+ gt$ '. h% &- -t4 ,d3 /6 .5 '. 0$7 nb) mR( k9- 6> (*%"'"#% r8cHURS5cgURS5S;agURS5b;[URS5[5(aURS5S::agONURS5b;[URS5[5(aURS5S::agOg[URS55S ::ag[URS 5[5(aURS 5S :agURS 5cgURS 5cgURS 5cgURS5cgURS5cgURS5cgURS5cgg)Nr>Fr?)rr@rrArIrrJrBrCrDrErFrGrHT)get_attr isinstanceintlenrXs r6 _check_selfShardingPass._check_selfus == ( 0 == ! 2 ==* + 7t}}->?EE01Q67 ]]8 $ 0t}}X6<<BBB  t}}^, - 24==7==$-- C C ==) * 2 ==0 1 9 ==/ 0 8 ==2 3 ; ==1 2 : ==* + 3 ==3 4 <r8cg)NT)rX other_passs r6_check_conflictShardingPass._check_conflictsr8cURS5Ul[URS5=(d URS55Ul[URS55Ul[URS55UlURS5Ul[URS55Ul[URS55UlURS 5Ul URS :dURS :aUR (dS 5e[URS 55Ul [URS 55Ul URS5Ul URS5nUR5UR5peURS5UlURS:Xa&[R R"R$nSnUR'XT5 UR(H6n UR+X5 UR-U 5 UR/X5 M8 UR1SUR25 UR5X5 g)Nr>r@rAr?rJrBrCrDrHrz3multiple comm stream need enable_overlap to be TruerErFrGrIrLbfloat16)r^ _dist_contextr`sharding_world_sizer?rJrBrCrDrHrErFrG global_blockrLrVarDescVarTypeBF16_build_sharding_groupsblocks_shard_optimizer_shard_gradient_synchronization_shard_parameterrPrW_optimization_pass) rX main_programstartup_programcontextrI main_block startup_block__amp_target_dtype____amp_target_dtype_name__blocks r6_apply_single_implShardingPass._apply_single_impls!]]>:#& MM+ , G h0G$  w/0 t}}];<"mm,<=%(7N)O%P"$' 6L(M$N!(, &) %  % % )T-F-F-J&& E &(+ MM3 4( $'* MM2 3' # $}}->?}}^4  % % '  ( ( *" {3 >>Z '#'<<#7#7#<#< (2 % ##J=!((E  ! !% 7  0 0 7  ! !% 7) )A)AB  >r8cHURU5 URX5 gN) _collective_data_parallel_groups_build_sharding_infos)rXr{rIs r6rr#ShardingPass._build_sharding_groupss --j9 "":&>> '(@(@'AAijrjyjyizz{| >??T%=%==B '(@(@'AAijrjyjyizz{| B##x~~5 !$"2"2!33abjbpbpaqqrs 5|$(@(@@ (\):(;;^_c_w_w^xxA @ !9!99*.'11A5550014444>>*a/K/2MNND$<$N>N2."3>!B&7&G#!)1?   .(  $$ M    & &} 5&-- Yz~~%> ?@GC*:::tzzA~ 11!4 ()@;+;+;C+@A 88DD))#E):**;U*CE OO "$''--"4J!+,Bjooc.B!CJ88DD"))*5 #5GG%%c:6GG&&uj9ww"<<"$'')"4#%#6#6q#9",//(";"--c->"::!0%*G$4 ' ' ' +W # ; !..OO '" O&NN3/%22%22 ..%.%7%7 #--c->oAr !!#r8cURS:ag/SQn[5n[5n[[UR55HupV[ U5(dMUR U;dM)URS5SnUSURS5nURU5(aMiURU5 UR S;dMURHn URU 5 M M [[[UR555H.upV[ U5(dMXS;dMURUSS9 M0 UHn URU SS9 M [[UR55GH7upV[ U5(dMUR S :XdM*/n URHnXt;dM U R!U5 M UR"R%SU 5 URSn [UR&5HupUR)X]-S -S S U /0S U /0SUR*R,SSS[.R0R2R4[6[8R:0S9nUR<R?URAU 55nM O URC5 g)Nr\)elementwise_mulsquared_l2_norm clip_by_normrr@GRAD)rrFrsumr all_reducexoutring_idr0z /gradient_clip_model_parallelism reduce_typerinputsrr)"r?rQrrr_is_gradient_clip_oprrrrrrrrrinput_arg_namesrr1rrS _insert_opridpaddle distributedReduceOpSUMrOpRoleOptimizerlrvarr)rXr{removed_op_typeremoved_op_idxremoved_tmp_varrr5rrrvarname reserved_vars sum_op_outputirnew_oprs r6_shard_gradient_clip!ShardingPass._shard_gradient_clipks ::> Q%Ijnn56GC'++ww/)XXc]1- '(B*//'*BC 88DD"&&s+ww"EE+-+>+>K+// <,?7 Yz~~%> ?@GC'++$%%c%6 A 'G  " "7 " 7'Ijnn56GC'++ww% " "$"4"4J!8%,,Z8#5!!#}5 " 3 3A 6 (1$2E2E(F$A'22! ) #m_5!& 8%}':':'='=*,N)6+=+=+F+F+J+J'  3 F**KK&NN=9)G,C7F !!#r8cURS:ag[[[UR555H up#[ U5(dM[ S5e UR5 g)Nr\z$weight decay is NOT supported by now)r?rrrr_is_weight_decay_oprr)rXr{rr5s r6r ShardingPass._shard_weight_decays\ ::> Yz~~%> ?@GC&r**): A !!#r8c&/n[[[UR555HupE[ U5(d OUR [ ;dM-SUR;de[URS55S:XdeURS5SnURU5(dEURURVs/sH nXv:wdM UPM sn5 URUSS9 MURRUR!U55 M [[[UR555HEupE[UR5S:XdM URSU;dM5URUSS9 MG UHQnUR#U5(aUR%USS9 UR#U5(dMAUR%USS9 MS UR'5 UR'5 gs snf)NParamrrFr)rrrrrr_supported_optimizer_type input_namesrarrextendrrrWr_get_param_gradhas_varrr)rXr{r|should_removed_optimizer_statesrr5rrs r6r,ShardingPass._shard_optimizer_ops_and_statess*,'Yz~~%> ?@GC!"%%ww33"..000288G,-222XXg.q1 88DD3::,.+>+>+>&4$+>))#E):,,33,,Z8%A, Y}/@/@%A BCGCB''(A-''*.MM((5(9 D7G!!'**&&wU&;$$W--))')> 7 !!#$$&5s H H c URS:dURS:agURGH0nURGHnUR UR 5(deUR UR 5(deUR SSU0SU0SURRSURUR 5[[R0S9nURS S [R-5 UR R#U5nUce[%UUR&UR(UR UR*S 9 GM GM3 UR-5 g) Nr\r broadcastrrrrootrr0/r)r?rErSrrr append_oprr get_var_rankrrr _set_attrr DataParallelrlrrrrrr)rXr{r|rrrparam_dist_attrs r6r)ShardingPass._insert_optimizer_broadcastssR ::>T99A= !00M&--!))%**5555$,,UZZ8888#--$<"EN!=#6#6#9#9 : :5:: F#V__ .   "C,*C*C$C&&GGN '222F#00#00&&,55 +.1: !!#r8cdXR;deURUnURU5$r)rTis_in_local_shard)rXrrs r6r)ShardingPass._is_parameter_in_local_shards5:::::55jA ..z::r8crXR;deURUnURU5nUceU$r)rTget_param_grad)rXrrp_gs r6rShardingPass._get_param_grad sC:::::55jA **:6 r8c (URS:agURVs/sHo"RPM nn[[ [ UR 555GHupE[XQ5(GaURS:Xa-URS5[RR:Xd=URS:XaJURS5[RR:XaSn[RRnOSn[RRnURSn[U5n UR U n [#UUUUU R$RU R'U 5UR(U5n UR*(aU R-U 5(dUR/US-SS9 OIUR1S UR2R5 UR1S S [4R6-5 [9XQ5(dGMUR:Sn [U 5n UR U n U R-U 5(aGM UR/USS9 GM UR=5 gs snf) Nr\rrrrrFrrr0r)r?rRrrrrr_is_param_grad_allreduce_oprr3distrrAVGr_get_base_name_from_grad_namerT_insert_reduce_oprrrlrUr rrrVrr _is_param_grad_sum_oprr) rXr{r dp_ring_idsrr5reduce_op_typerr base_namer reduce_oprs r6ru,ShardingPass._shard_gradient_synchronizations ::> -1^^<^Exx^ <Yz~~%> ?@GC*2::GG|+ .$--2C2CCGGx' .$--2C2CC%-N"&--"3"3K%-N"&--"3"3K//2 9*E $ = =i H -"!''**!..y9&&  //(::9EE))#')>LLD,?,?,B,BCLL&l.G.G(G%R44..q19(C $ = =i H $66yAA))#E):_Ab !!#e=sJc URS:agURVs/sHo3RPM nnURGHnUR U5unn/nUH?n XyS:XdM UR U 5UR :wdM.URU 5 MA [[[UR555GHup[U 5(aMU RGHn [XUR5(a)UR!U 5(dURU 5 MHX;aMOUR U 5n XR :XaU nO["R$"U S-5nUR'U 5nUR)UUR*UR,SS9nUR.R1U5n[3UR.UUR4UR6UR8S9 UR.R;U 5nUR=U 5nU R?X5 URAUU5 [CUU UUR U URDRU RGS5UR.5 GM GM [[[UR555H\upU RHS:waMU RSn U RJSnX;dM<URMU SS 9 UROUSS 9 M^ [[[UR555GH<up[QU RJ5S :XdeU RJSnU RHS :XaU RGS 5U;ahURR(aFUR U5UR :Xa'U RUS URRR5 O@URMU SS 9 O/UR U5UR :waURMU SS 9 MU RHS :wdGMUU;dGM UR U5UR :wdGM,URMU SS 9 GM? UHDn UR U 5UR :wdM$UROU SS 9 UROU SS 9 MF GM URW5 URW5 gs snf) Nr]rz @BroadCastF)rrr persistablerrcastrrrr),r?rRrrS"get_broadcast_vars_and_param_usager local_rankrrrrrrr_is_param_fp16_cast_op param_namesrrgenerater create_varrrrlrrrrrget_op_dist_attr_for_programget_input_dist_attr _rename_inputset_input_dist_attr_insert_init_and_broadcast_oprr3rrrrrarVrr)rXr{r|rrrneed_broadcast_vars param_usagenot_used_param_namerrr5r root_rankbroadcast_varname input_varnew_var ref_dist_attr op_dist_attrinput_dist_attrrs r6rvShardingPass._shard_parameterKs ::> -1^^<^Exx^ <!00M@@L #"$ ) +q0%22:>$//0(..z: *$D:>>)B$CD!"%%"$"4"4J." (A(A $@@LL/66zB !< - : :: FI $<$<<,6),7,@,@&5-)%/NN:$> ","7"7!2"+//"+//(- #8#!..OO )& * ..#)66)66%2%;%; !..KKBO%+7*J*J&+((G$88-2")%00!%++.. *** g#5 ED$D:>>)B$CD77f$//2  11!4 4))#E):**;U*CE$D=3D3D)E$FG2../1444 11!4 77k)wwy)[8 // - : :; G,77!8LLD4G4G4J4JK)44Su4E*66{C,778*44Su4EGG{*#{2%22;?$//0",,Su,=9H<* !..z:$//0**:E*B!--ju-E *o1~ !!#$$&C=sTcURS::agSUlSUlSUl[ UR 5S:XdS[ UR 5S35eUR Sn[ RRX5 URU5 URS:aCURS:XaURU5 O!URS :XaURU5 SSS5 g!,(df  g=f) Nrsharding_coalesce_grad_sharding_coalesce_param_rKzZgradient synchronization optimization only support one sharding group right now, but got [z].rr\r]) r?grad_coalesce_prefixparam_coalesce_prefixcomm_op_scheduling_priorityrarSrstatic program_guard_gradient_sync_optimizationrE&_fuse_overlap_parameter_comm_stage_two(_fuse_overlap_parameter_comm_stage_three)rXrxryrs r6rwShardingPass._optimization_passs ::? $=!%?"+-(4&&'1, hilmqnAnAjBiCCE F ,++A. ]] ( ( G  , ,] ;++a/::??? NZZ1_AA-PH G Gs A%C:: DcURS::aUR(dg[5R5n[ 5R5nUR UU5upEUR UUUU5 g)Nr)rFrBrrnr _group_grads_overlap_grad_comm)rXrr{r|coalesce_to_group_mapgrad_name_to_group_maps r6r@(ShardingPass._gradient_sync_optimizationss  & &! +T5H5H )+88: /1>>@ 8<8I8I  9 5    ! "  r8cR [5R5n[5R5n[XR5upE[ R S5 [ R SURS[UR55S[UR55S35 0nUR(a/Ul URRn[UR5HVnUS:Xa URn O [USS9n URR!U ["R$R&S .5 MX [ R S URS 35 0Ul[+UR55GHup[U 5S :de[U 5S :a[,R."UR0[3U5-5n UR5U U R6SSS 9 UR5U U R6SSS 9U lUR;SSU R<0U R<U R8S.SSSSSU R6[>[@RB0S9 OU R<SU l[ R SUS[EU R<V s/sHn [GU 5PM sn 5S35 [ RISUSU R<V s/sHoRJPM sn S35 U XjR8RJ'XR-n URU SnURU SnUR;SSU R80SU R80SURLS U RN[>[@RP0S9nXR(U'URSS!S"[TRV-5 UR(dGMkUURXl-UR\URXl/GM 0n[+UR`5GHunn[cU5(dMUReS#5SnUR<UnUUn SnUR(aURXRZn[UR55UR:a-[gU5nUR<UReS$5SnOSUR`XR- n[cU5(dS%5eUR<UReS#5SnUU/U/U4/UU'[U R<5S :dGM9U Rh(a)U R<S&nUUR!UU/U/U45 UUR!US -U/U R<U45 GM [kUR5SS'9nUHnUUSSS&2Hwunnnn[mUUUUURn[@RPS&/S(S(S)S*9 nUR(dMFUcMKUURXl-UR\URXl/My M URq5 gs sn fs sn f)+NzSharding Stage2 Optimization:zParam Bucket size is [], [z] Parameters are fused into [z ] BucketsrTforce_new_group comm_group comm_streamz#Parameter Communication would use [z ] streams.rrrr  stop_gradientcoalesce_tensorInputOutput FusedOutput copy_data use_alignrrzBucket[z] size [z]MB.z] parameters: .rOrPrrrrrr0rrParamOutz:Unexpected: sharding broadcast pre op should be broadcast.rKreverseFsharding_stage2_broadcast_depr is_recomputerr0)9rrnr group_paramrE_loggerinforakeysrBparam_comm_group_stream_pairsrrrangerCrrrSHARDING_STREAMrop_to_stream_idxrrr&r<strr'r coalesce_varrrrrForwardrrdebugrrrankrrrr rexecution_streamr=scheduling_priorityris_sharding_param_broadcast_opoutput_get_broadcast_first_depend_opr sortedr rlr)rXrr{r|group_to_param_mapparam_to_group_mapbroadcast_var_to_group_maprrr param_groupcoalesce_var_namepcomm_stream_idxrOrPrdep_mapr5r1 broadcast_var prior_varpre_op last_gradindicer prior_vars post_vars depend_ops r6rA3ShardingPass._fuse_overlap_parameter_comm_stage_twos)+88: /1>>@ 1< 772 .  45 $T%A%A$B$sK]KbKbKdGeFfgDEHI[I`I`IbEcDddm n &("    24D .!''--E45566)//E-eTJE2299&+'='M'M'S'S 7 LL5d6P6P5QQ[\ %'D !'(:(?(?(ABNA{#q( ((;!#$/$8$8..Q7%!((*%++ $"& ) ,6+@+@*%++ $"& ,A, ( ''*#[%5%56"-"2"2'2'?'? $T#T!2!2#V^^ ( ,7+;+;A+> ( LL!HS;CSCS)TCSa,q/CS)T%U$VVZ[  MM!NKRAS 5 [>RAS URS[ U 5S[ U5S35 0n0n0n0n/n[CU5GHun n[ UR(5S:GaGURE[FRH"URJ[MU 5-5URNSS S9Ul(URRRUUR(S5n[WURRURPURXURZUR\S9 UUUR*'UR2R_5nUUU'URaUR25 UR6(a:UR<R_5nUUU'URaUR<5 OUR(SUl(UR(HnUUURb'M UUURPRb'GM [ URe55n[ URe55n[ U5n URgU5n![ U!5S:XdeURgU 5n![ U!5S:XdeURgU 5n![ U!5S:Xde[i[k[CUR555GHoun"nU"U;aUU"nURSn U UR(SRb:Xd%SUR(SRbSU S35eURmU URPRb5 URoU URPRb5 U"U;aURqU"SS9 U"U;dMUU"nUR(SRbn#U#UR;dSU#SUS35eUR(Vs/sHnURbPM n$n/n%/n&UR(H:n'U'Rrn(U%RaU(5 U&R'[ U(55 M< URuU"SSU$0U$URPS.SSSS SURNS U%S!U&[v[xRz0S"9n)URRRUURP5n[}U)URZURXURRUR\S9 [UU"UR!UR.5URPURR[xRzS/SSS#S$9 n*GMr UR5 UU4$s snf)%a: conditions for gradients to be grouped: 1. group size < grad_bucket_size_numel 2. same dp group (TODO) 3. same src rank 4. same dtype 5. dependency: grad would NOT be used by other ops within group segment main logic: 1. record coalesce group 2. record all dp allreduce/reduce op idx 3. insert coalesce op 4. insert coalesce dependency (avoid allocate memory too early) 5. modify and remove allreduce/reduce op 6. ensure sharding-dp hybrid parallel logic gradients inside same group would be fuse into one coalesce tensor rNrc[URUR-5nURVs1sHo3RiM nn[ UR U55S:$s snf)Nr)rQrrrrra intersection)r5rvars_r var_namess r6op_depend_on_group5ShardingPass._group_grads..op_depend_on_groupsW**R-@-@@AE-2ZZ8ZcZI8u)))459 99sA%rrzZSharding should reduce grad first and than allreduce if Hybrid Sharding with Data-Parallelr\Trz@Hybrid Sharding with Data-Parallel should sync same gradient varz-Sharding Gradient Communication Optimization:zGradient Bucket size is [rKz] Gradients are fused into [z ] Buckets.FrQrrKz$Unexpected: it is supposed to sync [z ] but got [rrz-Unexpected: op is supposed to generate grad [rSrTrUrXrYrconcated_shapesconcated_ranksrsharding_grad_coalesce_depr_)ArrFr rVarGrouprQrarrr3rrrrrrr acceptablecollectrrcoalesce_op_idxr coalesce_dep_varnamerreduce_op_indicesrUr rrallreduce_op_indicesrbrcrr'rr&r;rirrjrlrrrrrpoprrrdrrrr*_rename_outputrrrrrBackwardrr r)+rXrrrfirst_backward_opr5first_backward_varname cur_group grad_groupsgrouped_grad_namesrr is_reduce grad_namerrmgrad_varjdep_op dep_varnamerHrGmodify_reduce_op_mapcoalesce_op_mapremove_reduce_op_indicesrr4last_reduce_op_idxlast_allreduce_op_idxgradcoalesce_op_set modify_op_set remove_op_setconflictrfirst_grad_name grad_namesrrgrad_r coalesce_oprs+ r6rEShardingPass._group_gradss0ii  & & * +,D ' Bb!!$&!  $ !2!C!CA!FT889   U : #c(lB)"--GGx/BGGM4JMM%%MM%%O5 !py//2 :9E $11*= 99Y/''$77$>>>%%h5&&y1 ()D)D EI%%h5y~~&!+01AI-A(U44Q)U44 QZF"("9"9!"?,33E4N4NO%*ZZ]" 49&tyy1#=B !%"4"4"9"9 :E/Ho224505578 45 "// >8}!!!"// >8}!!! --m<8}!!! Yuyy%9 :;GC**,S1//2  EJJrN$7$77:5::b>;N;N:O{[dZeefg7  E,>,>,C,CD!!)U-?-?-D-DE..  5 1o%',"'**Q-"4"4&"*=*==COCTT_`b_ccde=5:JJ?JDdiiJ ?"$!#"ZZE!KKE#**51"))#e*5( $;;*#Z0",','9'9 $U#T)?(.#V__ < &&&GG** G!..!..&&*33 9IIe889&&&&OO""'!=  w:XajUR0(dYURAS5(dCX;a/X'XRU S -UR$UR$SSUR,45 U S - n U S - n GM. [CU RE5SS9nUHnXSSS 2Hun nnnnnUS:XaSOSnUU UUURH[JRLS /SSUUS9 nUcME[OUURPURHURRS9 UcMtUUR,lUR4UR,lM M URT(Ga`SnURVU-S:XdeURnURnURXU-n URXU-n!UV"s/sHn"U"U-U :XdMU"PM n#n"[ZR]S5 [ZR]SURXS35 [ZR]SU#S35 [)U#5URVU-:XdeUV"s/sHn"U"U-U!:XdMU"PM n$n"[)U$5U:Xde[ZR]SU$S35 /n%/n&[ UR 5H5n'U%R[ U#SS95 U&R[ U$SS95 M7 [_[a[UR555GHup[U5(dMURS:Xa-URS5[<RR :XdeXn(U%U(n)U&U(n*UR#S 5SnUR(aUR,R2n[cURS55n+Sn,U+U-U :XaSn,U+U-n-UR9SU*R:5 UR9SU-5 U,(dGMU+U-n.UReU S -SSU0SU0SU)R:SU.S[<RR [f[JRL0S 9n/U/R9S!S"[hRj-5 UR(dGMWU/R,lUR4U/R,lGM URm5 gs sn"fs sn"f)#a overlap gradient communication with backward & optimizer computation. 1. assign gradient communications to grad comm stream 2. for coalesce gradient communication: 2.1 insert before communication dependencies 2.2 insert after communication dependencies only when need 3. there is not need to add explicit dependencies for non-coalesce gradient communication P.S. this overlap pass is ONLY adapted for standalone executor (graph based) and stream award allocator. NrTrLsharding_grad_comm_streamrNrrrOrPrrrKsharding_grad_comm_deprrrMsharding_reduce_avg_depr\F)rr`rr0skip_insert_when_sequential_run)rctxrz:Sharding Gradient Hierarchical Communication Optimization.zcurrent global rank idx: rZzlocal inter node ranks idx: zlocal intra node ranks idx: root_idrrrr0r)7rBgrad_comm_group_stream_pairsrrrfrDrrrrrrr3rrrrrqrjrrarrrUr rnr=rorrrrr^rsrdr rlrrrrrrHrmrJrbrcrrr`rrrr r)0rXrrrGrHrrrstreamrr{reduce_op_countgrad_comm_op_to_stream_idxrr5 stream_idxrOrPreduce_varname grad_grouppost_idxnext_oprrrr0rrrnranks_per_node global_group global_ranksrelative_idx_in_nodenode_idxrminter_node_ranksintra_node_ranksinter_node_groupsintra_node_groups_grad_comm_stream_idxinter_node_groupintra_node_groupdst_rankin_peerintra_node_dstinter_node_dstrs0 r6rFShardingPass._overlap_grad_comms;&"" ,.)##))t001AAv%++)%F14F  - - 4 4"'#) 2ii%'" ~GC)"--77l*rww}/E&&//33J0,/H/HH 1;2.!>>zJ  #?? K! "$5!1!!42>B !..33~EEEz'!+  &OOB/&33'4LL  $GL #QwH..:3O3O A L''$&33&OO'4LL  1< -44 0 Y 6**z/K/K!'lG"<<<7GLL%=**3377= #>>%03~EEE:EG%%688%%91HCGGx' .$--2C2CC&88 MM*JKK)') L''!G&33&33 5LL   1$ 1HCO&T 5ADbD!*-FFED09&&OO""'!-4S (>!%.%;%; ..!*!3!3  #.?J ++< <<"++?="V  ( ( ( O++o=B BB(..L'--L#'#3#3o#E ''?:H) (D/)-AA(   LLL  LL4T5E5E4FaH I LL78H7IK L$%++>? ? ) (D?*h6(   '(O; ;; LL78H7IK L "  " 4445!((%&6M"((%&6M 6$D599)=$>?-b118+GGM2dmm6G6GGH,F+I('89M'N$'89M'N$%'YYu%5a%8N**&(ll&C&C "2779#56H#G/15II"&%-%?NLL,<,?,?@LLN;w)1_)D!&!>!>!G!)$'#8 %~%!*+;+>+> )> -t}}/@/@ +V__ #"? "((*C,2K2K,K ...@KF,,= $ @ @#,,@[@b e  s `+`+ `0(`0)rlrLr=rRrHrBrJrFr;rrDrhrVrEr<rerCrGrUrSrmrWr?rT)__name__ __module__ __qualname____firstlineno__rOrcrhrrrrrrtrrrrrrrrurvrwr@rArBrErF__static_attributes__ __classcell__)rYs@r6r;r;[s&0&P3?j=*/Jb E=$~B$H $&'P!$F; 6$pE'NQ2 "o$b |=|``r8r;cnURHnUR[;dMUs $ [S5e)NzCould not find optimizer op.)rrr Exception)rr5s r6rrrrs0ii 77/ /I 2 33r8c URU5nURU5n URUSSU0SU0SUSU[U0S9n U R SS[ R -5 [U U RU RUU RS 9 X4:wahURUS S UR0S URS UR[U0S9n [U U RU RUU RS 9 gg)z empty op for initialization rrrrrrr0rremptyrrrrN)rrrrrrr rrrrrrr) r insert_idxrr#r0rrr>r|broadcast_var_dist_attrrs r6r,r,s-IIg&M*KK * * W~ w I  + F ^S<+D+D%DE:,,,,(11 .. M../,,,,W /   ?  # 0 0 # 0 0 ,55  r8c LUS:d SU35eURUUSU/0SU/0SUSUSU[U0S9n URURU55n [ U U R U R UU RS 9 U RS S [R-5 U $) Nrz5root id should be a positive int, but now root id is rrrrrrrr0r) rrrrrrrrrrr ) rop_typer reduce_varrrr>rrrrs r6rrs a< ?yI< * * j\" % w w ;   + F== *I;##  ^S<+D+D%DE Mr8c`SnSn[U5U-U/n[XX25n[XXB5nXg4$Nrr)rar) origin_groupsharding_group_sizermdp_axis sharding_axisrrrs r6rr;sCGM  "5 57J KE|GBH$\-NN  ##r8cURRS5=(a* URRS5RS5$)Nr0z/gradient_clipr1r2r3 startswithr4s r6rrFs; 77  N + # 1j!"#r8cURRS5=(a* URRS5RS5$)Nr0z/regularizationrr4s r6rrLs; 77  N + $ 1j"#$r8cb[U5(dg[X[[RR R 5(dg[X5(agURSnUSURS5nURU5(dgURU5R$)NFrr) r _is_desired_cast_opr}rrorpFP32rrrrr is_parameter)rr5rrs r6rrRs "    ')=)=)B)B  u))%%a(K3k..s34I == # # 99Y  , ,,r8cr[U5(ag[X5(dgURSnX2;agg)NFrT)rrr)rr5rrs r6r$r$bs:b u ) )##A&J r8cLURS:wag[UR5S:Xde[UR5S:XdeUR URS5nUR URS5nUR U:wdUR U:wagg)Nr!FrrT)rrarrrr)rr5 src_var_type dst_var_typer2 output_vars r6rrms  ww& r!! "a '' ' r"" #q (( ( ",,Q/0I2..q12J,&**:*:l*J r8cSnSU;aUSURS5nU$SU;aUSURS5nU$SU;aUSURS5nU$)Nz.cast_fp16@GRADz.cast_bf16@GRADr)r)rrs r6rrs}II%A /@ AB  i 'A /@ AB   I 7 w 78 r8c[U5(dgURSn[U5nURU5(dgUR U5R $)NFr)rrrrrrr5rrrs r6rrsQ %b ) )%%a(K-k:I == # # 99Y  , ,,r8c[U5(dgURS:wagURSn[U5nUR U5(dgUR U5R $)NFrr)r rrrrrrrs r6rrs` "   ww%%%a(K-k:I == # # 99Y  , ,,r8cURS:H=(aN URRS5=(a, [RURR S5;$)Nrr0)rr1r2rr r3r4s r6rprpsL ; F GG  ^ , F  % %n)E Er8cSnURHn[XQRU5(aM URU5nURnUR U5nUR n [U5S:XaMkUSn U S:dMxXS:dMUbX:XdM[URUR U U5n [U 5n U$ U$)NrrKr) rrrr(rget_input_dims_mappingrrar process_idsr) rank_idr5r>rrrrrinput_dim_mapping mesh_shapebatch_size_axis group_rankss r6rrsH(( $J,GG$AA"EI$11L ) @ @ L %++J$%*/2O# (Ca(G?o&@"1$00$**' #K 1=H O1)0 Or8c0nSn/nUH$n[U5nX6- nURXV45 M& [U5Vs0sHow/_M nnSnSn UH2upVXS-US--U- :aUS- nX(RU5 X- n M4 U$s snf)z shard the continuous param into same rank and divide the forward&backward computation into segment, which will favor the fuse pass in later. we assume that the params is already sorted by utilization order. grg?r)rrrf) r group_sizemappingtotal_param_mem param2memrmemrcur_rankmem_accus r6partition_by_use_orderrsGOI5!%&$J/0/"u/G0HH  +x!|*partition_by_greedy_even..sAEr8rzparam [z#] should larger than 0, but it is [r)rfindexminrrrr)rrr rank_sizesrrmnumels r6partition_by_greedy_evenrsGz"# C* E{{3u:& U#)5;;:qy ejj\!DUG1 M y  u  Nr8cUS:Xa [X5nO [X5n[RS5 UR 5HwupE[RSUS[ UVs/sHn[ U5PM sn5S35 [RSUVs/sHofRPM snS35 My U$s snfs snf)N greedy_evenzSharding Parameter Partition:zRank:z, Parameter Size:z MB.zParams in this rank: rZ)rrrbrcitemsrrr)rralgorrank_to_paramskvrs r6partition_parametersr"s 1&E/C LL01$$& A3'!,L!3\#->!,L(M'Nd S   ,!-D!3hh!-D,EQGH '  -M-Ds $CC c0nUHupEXE4X4R'M /nURHOnURH"nX;dM X;dMURU5 M$ [ U5[ U5:XdMO O URSn 0n [ UR5n /n [ U 5(GabU R [;GaM[[[UR555HbupUR [;dM[ URS55S:XdeXzURS5S'U RU 5 Md [ U5[ U 5:XdeUH\nURSS9nURRXR5 URUUR!X55 M^ U Hn UR#U SS9 M UR%5 [ UR5U :Xde[&R)S US 35 UVs/sHoCUPM sn$s snf) NrKrrrnop)rFrz(Sharding the Order of param being used: rZ)rrrrrarrrrrrrrr1 copy_fromset_op_dist_attr_for_programr(rrrbrc)r param_gradsr>pname_to_pg_pairsryg use_orderr5rlast_op pname_to_opnum_opsremove_op_indicesrpnamers r6rr s %&F&&!Iii,,J/+  , - y>S!23 3 iimGK%))nGg7<<3L#LYuyy%9 :;GCww33288G,-22246BHHW-a01!((- < 9~[!1111E__%_0F KK ! !+"4"9"9 :  5 599+:LM %C   Su  -% 599~((( LL;I;aHI*3 4)Qa ) 44 4s6Ic0n0n/n[U5nURHnURUR5nUR Xg5(aUR Xg5 O[U5nUR Xg5 UR UR5UlXR;aX%RUR5 OUR/X%'XSUR'M X#4$)z- param are group by: rank id fuse_size dtype )rrrrrrr r)r fuse_sizertrubucketrrrms r6rara@s F#I%%))%**5    , ,   e * +I   e *&3&E&E JJ' #  *  ) 0 0 <-2ZZL  ))25::&%&(  11r8c8\rSrSrSrSrSrSrSrSr Sr g ) ribcXlUVVs0sHupVURXV4_M snnUl[UR5[[ UR55:XdS5eUVVs/sHupWUPM snnUlUR Vs/sHoURPM snUlURUlX l URRUR5Ul X@l [UR URUR5Ul0UlUR#5 gs snnfs snnfs snf)Nz&found duplicated param in params_grads)rrrIrarQrr%rrrJrrr#rGr"r param_to_rank_map_param_to_rank)rXrrmrIrGryr)rs r6rOShardingInfo.__init__cs  8DE QVVaV^ E4$$%S1B1B-C)DD 4 D&22\TQq\2 ,0KK8KqFFK8,,++++D,<,<=.2 KK$*>*>   !#F 38sE0EE cURR5H&upUHnXRUR'M M( g)z0 mapping parameters to the rank which holds it. N)rrr5r)rXrmrrs r6r6ShardingInfo._map_param_to_rankxs;!//557LD15""5::. 8r8c@XR;aURU$g)NrK)r5)rXrs r6rShardingInfo.get_var_ranks" (( (%%g. .r8c>URU5UR:H$r)rr#rXrs r6r ShardingInfo.is_in_local_shards  ,??r8c[5n[5n0n[RURS5nURHEn[ U5(aMUR H nXpR;dMXW==S- ss'M" MG URHn[XUR5(dM UR SnURSnURU5 URU5 XtU'XW==S-ss'URUURU'M UR5HupU S:dM URU 5 M X%4$r) rQdictfromkeysr%rrrr$rrr5r) rXrbroadcast_vars fp16_params fp16_to_fp32r.r5rrrusages r6r"/ShardingInfo.get_broadcast_vars_and_param_usages9e  mmD$4$4a8 ))Bb!! 00 !1!11+q0+1))B)%T5E5EFF++A.J--a0K   { + OOK ((2 %  #q ( #.2.@.@.LD  { +(--/LEqy""5)0**r8cURU5(d[SUS35eXR;a[SUS35eURRUS5$)Nzparam[z] not in current rank.z] not in params_grads)r  ValueErrorrIgetr=s r6rShardingInfo.get_param_gradsb%%j11vj\1GHI I .. .vj\1FGH H  $$Z66r8) rJrrr#r%r5rrIrGrN) rrrrrOr6rr r"rrrfr8r6rrbs!"*6 @+:7r8rc,\rSrSrSrSrSrSrSrg)ricXlSUlSUlSUl/UlSUlSUlSUl/Ul/Ul SUl g)NrKrF) max_sizerrmrrrjrrrrr )rXrMs r6rOVarGroup.__init__sS      $(!#!#$&!!&r8cURS:XagURUR:wagX R:wagUR[U5-UR:agg)NrTF)rrrmr rMrXrrms r6rVarGroup.acceptablesP ::?{{djj(yy zzM%004==@r8cURUlX lU=R[U5- slURR U5 gr)rrmrr rrrPs r6rVarGroup.collects7[[   mE**  r8c,[UR5$r)rarrbs r6__len__VarGroup.__len__s499~r8) rrrrjrr rMrrmrrN) rrrrrOrrrUrrfr8r6rrs '  r8rr)r)Ologging functoolsrrpaddle.distributedrr8paddle.distributed.auto_parallel.static.operators.commonrrr5paddle.distributed.auto_parallel.static.process_groupr-paddle.distributed.auto_parallel.static.utilsrr r r r r rrrrr7paddle.distributed.fleet.meta_optimizers.sharding.utilsrpaddle.frameworkr paddle.staticrr paddle.utilsrauto_parallel_master_gradr pass_baserr pass_utilsrop_proto_and_checker_makerrkOpRoleAttrNamerrrINFOrbrorpFP16r}r~r7r;rrr,rrrrrrr$rrrrrrprrrr"rrarrrfr8r6rhsy !     Q!G$>.. ( ( / /--==?     W\\ "||++00%C'(r8r)rj+46 D OO&R$# $ - %%**% & - -(,>4(  15h2DM7M7`!!r8