
    Αi/<                        S r SSKJr  SSKrSSKrSSKJr  SSKr\(       a  SSKJr  SSK	J
r
  SSKJrJr  / r " S S	5      rg)
zParameter Server utils    )annotationsN)TYPE_CHECKING)Tensor)RoleMakerBase)ExecutorProgramc                  x    \ rS rSrSr  S     SS jjr  S         SS jjrS rSS jrSS jr	S	 r
S
rg)DistributedInfer    z6
Utility class for distributed infer of PaddlePaddle.
Nc                   U(       a  UR                  5       U l        O1[        R                  R	                  5       R                  5       U l        U(       a  X l        O#[        R                  R                  5       U l        S U l        g N)cloneorigin_main_programpaddlestaticdefault_main_programorigin_startup_programdefault_startup_programsparse_table_maps)selfmain_programstartup_programs      f/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/fleet/utils/ps_util.py__init__DistributedInfer.__init__%   sj    
 '3'9'9';D$ 224::< $ *9' 557 ' "&    c                   SSK Jn  UR                  R                  GcD  UR                  US9  [        R
                  R                  5       nUR                  5       nSUl        UR                  XgS9nUR                  X R                  S9  UR                  5       (       a   UR                  US9  UR                  5         ONUR                  [        R                   R#                  5       5        UR%                  5         U R'                  X5        [        R                   R#                  5       n	U R                  n	[        R                   R)                  5       n
U R*                  n
g g )Nr   fleet)
role_makerT)strategy)r   )dirname)paddle.distributedr   _runtime_handleinitr   	optimizerSGDDistributedStrategya_syncdistributed_optimizerminimizer   	is_serverinit_server
run_serverrunr   r   init_worker_init_dense_paramsr   r   )r   exelossr    r"   r   fake_optimizerr!   r&   global_startup_programglobal_main_programs              r   init_distributed_infer_env+DistributedInfer.init_distributed_infer_env9   s&    	-;;&&.JJ*J-#--113N002H"HO33 4 I &A&A     !!'!2  "==?@!!#''5%+]]%J%J%L"%)%@%@""(--"D"D"F"&":":/ /r   c                B   SSK Jn  U R                  c  0 U l        UR                  R                  R                  nUR                  5        HF  u  p4UR                  (       a0  UR                  S5      nUR                  5       U R                  U'   MF  MH     U R                  $ )Nr   r   z@GRAD)	r#   r   r   r$   	_send_ctxitems	is_sparsestriptable_id)r   r   send_ctxgradnamectxparams         r   _get_sparse_table_map&DistributedInfer._get_sparse_table_map[   s}    ,!!)%'D"{{22<<H!)!1==$NN73E47LLND**51 "2 %%%r   c           
     t   U R                  5       nUGb  UGb  U R                  R                  5        Vs/ s H5  n[        R                  R
                  R                  U5      (       d  M3  UPM7     nnU Vs/ s H#  nUR                  U;  d  M  UR                  U4PM%     nnU Vs/ s HN  n[        R                  R                  [        R                  R                  X$S   5      5      (       d  MI  US   PMP     nn[        R                  R                  UUU R                  US9  g g g s  snf s  snf s  snf )Nr      )r   vars)rC   r   	list_varsr   r   iois_persistablenameospathisfilejoin	load_vars)r   r2   r"   r   vall_persist_varsdense_persist_varsneed_load_varss           r   r1   #DistributedInfer._init_dense_paramsi   s(    6683? 11;;= =A==##2215 =    *")A66!22 )  " ,+A77>>"'',,w!"=> !+  
 MM##!55#	 $ ! $3 
"
s$   2D++D+7D0D0'AD53	D5c                ^    U R                  5       nU R                  U R                  U5      nU$ r   )rC   _convert_programr   )r   varname2tablesconvert_programs      r   get_dist_infer_program'DistributedInfer.get_dist_infer_program   s3    335//$$n
 r   c                $   ^ U4S jnU" U5      nU$ )Nc                ~   >^ ^ SSS.mU4S jnU U4S jnU" T 5      n[         R                  " S5        U" T U5        T $ )NW)lookup_tablelookup_table_v2c                J  > 0 nU R                  5       R                   H  nUR                  TR                  5       ;   d  M#  UR	                  S5      SL d  M9  UR                  TUR                     5      S   nUR                  U/ 5      nUR                  U5        XAU'   M     U$ )Nremote_prefetchTr   )global_blockopstypekeysattrinputgetappend)_programpull_sparse_opsop
param_namerd   SPARSE_OP_TYPE_DICTs        r   _get_pull_sparse_ops]DistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._get_pull_sparse_ops   s    "$"//155B#6#;#;#==GG$56$>%'XX.A"''.J%KA%N
-11*bA

269
3 6 '&r   c                .  > S nUR                  5        GHm  u  p4TR                  5       R                  nU Vs/ s H2  nTR                  5       R                  UR	                  S5      S      PM4     nnTR                  5       R                  US   R	                  S5      S      nUR
                  TR                  5       ;  a  [        SUR
                   S35      eTUR
                     n	US   R                  S5      n
US   R                  S5      nUS   R                  nU Vs/ s H2  nTR                  5       R                  UR                  S	5      S      PM4     nnU" TX}5        U Vs/ s H  oeR                  U5      PM     nnUS S S
2    H"  nTR                  5       R                  U5        M$     S
/[        U5      -  n[        TR                  5       R                  5      S-   /[        U5      -  n[        TR                  5       R                  5       GH   u  p[        S[        UR                   5      5       HZ  nUR                  UR                   U   5      n[        U5       H*  u  nnUR
                  U;   d  M  [#        UUU   5      UU'   M,     M\     [        S[        UR$                  5      5       HZ  nUR	                  UR$                  U   5      n[        U5       H*  u  nnUR
                  U;   d  M  ['        UUU   5      UU'   M,     M\     GM     ['        U5      [#        U5      -
  S:  a<  [#        U5      S-   nTR                  5       R)                  USXxS.SU0UU
U	SUS.S9  GMe  [        S5      e   g s  snf s  snf s  snf )Nc                |   U R                  5       n[        UR                  5      nSnS/[        UR                  5      -  nS/[        UR                  5      -  n[        UR                  5       GH  u  p[	        S[        U	R
                  5      5       Ha  n
Xh   S:X  a    OXU	R                  U	R
                  U
   5      n[        U5       H'  u  pUR                  U;   d  M  SXh'   [        XX5      n  M_     Mc     [	        S[        U	R                  5      5       Hb  n
Xx   S:X  a    M  U	R                  U	R                  U
   5      n[        U5       H'  u  nnUR                  U;   d  M  SXx'   [        XH5      nM)     Md     GM     [	        [        UR                  5      5       H-  n
Xj   S:X  d  M  Xz   S:X  d  M  [        R                  " S5          g    XE:  Ga  / n[	        US-   [        U5      5       H-  n
Xj   S:X  d  M  UR                  UR                  U
   U
45        M/     [        U5       GH  u  p/ n[        5       nUR                  U	S   5        UR!                  U	S   5        SnU[        U5      :  Ga  UU   nUR                  U   n	/ n[	        S[        U	R                  5      5       H2  nU	R                  U	R                  U   5      nUR                  U5        M4     [	        US-
  US-
  S5       H  nUR                  U   nUU;   a  M  Sn[	        S[        UR
                  5      5       Hd  nUR                  UR
                  U   5      n[	        [        U5      5       H"  nUU    H  nUU;   d  M  Sn  O   U(       d  M"    O   U(       d  Md    O   U(       d  M  UU   (       a  [        R                  " S5            g UR                  U5        UR!                  UR                  U   5        GM     US-   nU[        U5      :  a  GM  UR#                  5         U H  nUR$                  R'                  U5      nUR)                  UR                  U   R$                  5        UR$                  R+                  US-   US-   5        UUR                  U   l        UR                  R-                  U5      nUR-                  U5      n UR-                  U5      n!UR                  R/                  UU5        UR/                  UU 5        UR/                  UU!5        US-   nGM     GM     UR$                  R1                  5       [        UR                  5      :X  d   e[	        [        UR                  5      5       H;  n
UR$                  R3                  U
5      UR                  U
   R$                  :X  a  M;   e   g g )	Nr   rF   zunable to re-arrange dags order to combine distributed embedding ops because a op both needs embedding table's output as input and produces ids as the same embedding table's inputFTzDunable to re-arrange dags order to combine distributed embedding ops   )rc   lenrd   	enumeraterangeoutput_namesoutputrK   maxinput_namesrh   minwarningswarnrj   setaddsortdesc
_insert_op	copy_from
_remove_oppopinsertop_sizerm   )"programinputsoutputsrc   min_output_indexmax_input_indexinput_indexesoutput_indexesidxrm   ioutsin_idin_varinsout_idout_varmove_opsqueuevisitedstartpos	op_inputskjop1foundtyindexr   	insert_opinput_stateoutput_states"                                     r   dag_check_up_and_reorder|DistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._pull_sparse_fuse.<locals>.dag_check_up_and_reorder   sX   #*#7#7#9L'*<+;+;'<$&(O%&C#l.>.>*?$?M&'S3|/?/?+@%@N#,\-=-=#>!&q#boo*>!?A,1Q6 %#%99R__Q-?#@D1:61B#);;$#69:M$669/6OO$)	 2C	 "@ "'q#bnn*=!>A-2a7 %"$((2>>!+<"=C3<W3E#*<<3#6:;N$77:(88&$4 4F	 "? $?, #3|'7'7#89(+q0^5F!5K$MM !V # : (9#%!&,q0#m2D"A  -/14 (1A1A!1Da0H I	"
 &/x%8EA$&E&)eG!LLA/#KK1.$%E"'#e*"4&+El%1%5%5c%:,.	).q#bnn2E)FA*,((2>>!3D*EC$-$4$4S$9 *G */$'!G-=-A2*"A +7*:*:1*=C'*g~(0,1E-21c#:J:J6K-L/2zz#:J:J1:M/N16s9~1FA5>q\349<@E49 6B 05u05 2G ,15,1 .M (-u+9!+<,4MM0v-. -3(-Q(/L4D4DQ4G(H3*"4 ).	C #(#e*"4F "JJL).'3'8'8'C'C$4(" !%|/?/?/F/K/K L , 1 1 < <$)AIuqy!" @D 0 0 7 <,8,<,<,@,@,G	.;.?.?.F/=/A/A%/H , 0 0 7 7$4i!" !. 4 4$4k!" !/ 5 5$4l!" 4Da3G 0+ */U &9B  ,0088:c(,,?      "'s<+;+;'<!=A , 1 1 4 4Q 7#/#3#3A#6#;#;!<!< ">W :r   Idsr   r^   zcan not find variable z!, please check your configurationpadding_idxis_distributedOutrt   rF   distributed_lookup_table)r   r^   OutputsT)r   r   r>   is_testlookup_table_version)r   re   r   r   attrsz9something wrong with Fleet, submit a issue is recommended)r;   rc   rd   rG   rh   rK   rf   
ValueErrorrg   re   rz   r   r   rv   rw   rx   ry   r{   r|   r}   r   )rk   rl   r   rB   rd   all_opsrm   r   wr>   r   r   op_typer   op_idxsr   inputs_idxsoutputs_idxsr   r   r   r   r   r   r   distributed_idxr   rX   s                             r   _pull_sparse_fuseZDistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._pull_sparse_fuse   sz   rh #2"7"7"9JE%22488G #&"%B  ,,.33BHHUOA4FG"%  
  ,,.33CFLL4Ea4HIAvv^%8%8%::(4QVVH<]^   .aff5H"%a&++m"<K%(V[[1A%BN!!fkkG #&"%B  ,,.33BIIe4DQ4GH"%  
 -WfF;>?3R}}R03G?&tt},,.99#>  - $&$V"4K$'(<(<(>(B(B$Ca$G#H3L $L $-W-A-A-C-G-G#H!&q#boo*>!?A#%99R__Q-?#@D1:61Bv#);;$#69<(+[-?:&K$6 2C "@ "'q#bnn*=!>A"$((2>>!+<"=C3<W3E#*<<3#6;>(+\&-A<&L$8 4F "? $I  <(3{+;;q@*-k*:Q*>,,.99"1!;+1#:%.$82@/:,4+/8?# :  )W M #:$ @s   9N'9N/NzBlookup_table will be forced to test mode when use DistributedInfer)r~   r   )r   rp   r   rl   ro   rX   s   `   @r   distributed_ops_pass?DistributedInfer._convert_program.<locals>.distributed_ops_pass   sF    363"O'}~ 37;OMMT g7Nr    )r   r   rX   r   covert_programs     `  r   rW   !DistributedInfer._convert_program   s    T	l .l;r   )r   r   r   )NN)r   Program | Noner   r   returnNone)
r2   r   r3   r   r    zRoleMakerBase | Noner"   z
str | Noner   r   )r   r   )__name__
__module____qualname____firstlineno____doc__r   r7   rC   r1   rZ   rW   __static_attributes__r   r   r   r
   r
       s     (,*.&$& (& 
	&0 ,0" ; ;  ; )	 ;
  ; 
 ;D&4Xr   r
   )r   
__future__r   rL   r~   typingr   r   r   (paddle.distributed.fleet.base.role_makerr   paddle.staticr   r   __all__r
   r   r   r   <module>r      s6     " 	    F/
B Br   