
    Αi                    4   S SK r S SKrS SKJr  S SKrS SKJr  S SKJrJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJr  S S	K7  S S
KJr  S SKJrJrJr  / SQrS rS rS r " S S5      r " S S\5      r " S S5      r " S S\5      r  " S S5      r! " S S5      r" " S S\"5      r# " S S\"5      r$ " S  S!\"5      r% " S" S#\%5      r& " S$ S%\"5      r' " S& S'5      r( " S( S)\(5      r) " S* S+5      r* " S, S-\*5      r+ " S. S/5      r, " S0 S15      r- " S2 S3\5      r.g)4    N)text_format)fleet)CommunicatorHeterClient)wait_server_ready)the_one_ps_pb2)RuntimeBase)Coordinator)*)core)CompiledProgramExecutorProgram)TableSparseTableGeoSparseTableBarrierTableTensorTable
DenseTablec                 l    U S   n[        U5       H   u  p4[        U5      U:X  d  M  X@S   U   U4s  $    g)Norigin_main_programsorigin_startup_programs)NNN)	enumerateid)context
program_idprogramsiprograms        `/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/ps/the_one_ps.pyget_program_by_idr!   +   sE    -.H)
g;*$$=>qA1DD *     c                    [        X!5      u  p4nUR                  5       R                   H  n[        U5      (       d  [	        U5      (       d  M%  UR                  S5      S   nXp:X  a  UR                  S:X  d  UR                  S:X  d  M`  UR                  S5      (       a(  UR                  S5      S:w  a  UR                  S5      s  $   g   g )NWr   lookup_tablelookup_table_v2table_classnoneMemorySparseTable)	r!   global_blockopsis_distributed_sparse_opis_sparse_opinputtypehas_attrattr)varnamer   r   main_programstartup_programidxop
param_names           r    parse_table_classr8   3   s    ):7)O&L3'')--'++L4D4DXXc]1%
 !>)ww++{{=))bggm.D.Nww}--* .r"   c           	         [        X25      u  pEnSnUR                  5        HB  nUR                  U:X  d  M  UR                  S   n[	        SU SU SU R
                   35          O   U R
                  n	U R                  S:X  a  XS-   :w  a  [        SUS-    SU	 35      eOX:w  a  [        S	U SU	 35      eU R                  n
U R                  S:X  a  XS-
  :w  a  [        S
US-
   SU
 35      eg XS-
  :w  a  [        SUS-
   SU
 35      eg )Nr      z	new var: z, SparseAccessor   z;The fea_dim is wrong, it will be sparse_embedding_dim + 2: z
, but got z7The fea_dim is wrong, it will be sparse_embedding_dim: z>The embedx_dim is wrong, it will be sparse_embedding_dim - 1:    z>The embedx_dim is wrong, it will be sparse_embedding_dim - 3: )	r!   	list_varsnameshapeprintfea_dimaccessor_class
ValueError
embedx_dim)accessor_protor2   r   r   r3   r4   r5   embedding_dimvarrB   rE   s              r    check_embedding_dimrI   F   sp   ):7)O&L3M%%'88wIIaLMIcU"]O2n6L6L5MNO	 ( $$G$$(88a''Mm^_N_M``jkrjst  (
 #I-Xbcjbkl   **J$$(88**PQ^abQbPccmnxmyz  +
 **PQ^abQbPccmnxmyz  +r"   c                        \ rS rSrS rS rSrg)Serviceh   c                     g N selfs    r    __init__Service.__init__i       r"   c                 J    SUl         SUl        SUl        SUl        SUl        g )NBrpcPsServerBrpcPsClientBrpcPsServicer      )server_classclient_classservice_classstart_server_portserver_thread_numrQ   service_protos     r    _setService._setl   s*    %3"%3"&5#*+'*,'r"   rO   N__name__
__module____qualname____firstlineno__rR   ra   __static_attributes__rO   r"   r    rK   rK   h   s    -r"   rK   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )
GpuServicet   c                 "   > [         TU ]  5         g rN   superrR   rQ   	__class__s    r    rR   GpuService.__init__u       r"   c                      SUl         SUl        g )NPsLocalServerPsLocalClient)rZ   r[   r_   s     r    ra   GpuService._setx   s    %4"%4"r"   rO   rd   re   rf   rg   rR   ra   rh   __classcell__rp   s   @r    rj   rj   t   s    5 5r"   rj   c                        \ rS rSrS rS rSrg)Accessor}   c                 <    SU l         S U l        SU l        SU l        g )N r   )rC   	optimizerfeature_dimrG   rP   s    r    rR   Accessor.__init__~   s!     r"   c                    [        XC5      u  pgnSn	UR                  5        H$  n
U
R                  U:X  d  M  U
R                  S   n	  O   UR	                  S5      (       d  US   (       a  SUl        OSUl        UR	                  S5      (       d!  UR
                  S:X  a  U	S-   Ul        OXl        UR	                  S	5      (       d%  UR
                  S:X  a  U	S-
  Ul        O
U	S
-
  Ul        UR	                  S5      (       d  SUl        UR                  nUR	                  S5      (       d  SUl
        UR	                  S5      (       d  SUl        UR                  nUR	                  S5      (       d  SUl        UR                  S:X  a  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR	                  S5      (       d  SUl        UR.                  UR0                  4 GH  nUR	                  S 5      (       d6  UR
                  S!:X  a  S"Ul        UR
                  S#:X  a  S$Ul        OS$Ul        UR                  S%:X  d  UR                  S&:X  a  UR2                  R	                  S'5      (       d  SUR2                  l        UR2                  R	                  S(5      (       d  S)UR2                  l        UR2                  R	                  S*5      (       d  S+UR2                  l        [;        UR2                  R<                  5      S:X  a'  UR2                  R<                  R?                  S,S-/5        UR                  S":X  Ga   UR@                  R	                  S'5      (       d;  URB                  S.   RE                  S/5      S   n[G        U5      UR@                  l        UR@                  R	                  S*5      (       d;  URB                  S   RE                  S/5      S.   n[G        U5      UR@                  l        [;        UR@                  R<                  5      S:X  a'  UR@                  R<                  R?                  S,S-/5        UR                  S$:X  d  UR                  S0:X  d  GM  URH                  R	                  S'5      (       d;  URB                  S.   RE                  S/5      S   n[G        U5      URH                  l        URH                  R	                  S*5      (       d;  URB                  S   RE                  S/5      S.   n[G        U5      URH                  l        URJ                   Vs/ s H  nURE                  S/5      PM     nnURH                  R	                  S15      (       d1  UR
                  S#:X  a!  [G        US   S   5      URH                  l&        OS2URH                  l&        URH                  R	                  S35      (       d1  UR
                  S#:X  a!  [G        US   S   5      URH                  l'        OS4URH                  l'        URH                  R	                  S55      (       d1  UR
                  S#:X  a!  [G        US   S   5      URH                  l(        OS6URH                  l(        [;        URH                  R<                  5      S:X  d  GM  URH                  R<                  R?                  S,S-/5        GM     g s  snf )7Nr   r:   rC   
use_ps_gpuCtrDymfAccessorr;   rB   r<   rE   r=   embedx_thresholdnodeid_sloti0#  feature_learning_rateg?	zero_initTFnonclk_coeffg?click_coeffg      ?base_thresholddelta_thresholddelta_keep_days   show_click_decay_ratedelete_thresholddelete_after_unseen_days   ssd_unseenday_thresholdr?   sgdSparseNaiveSGDRuleadamSparseAdamSGDRuleSparseAdaGradSGDRuleStdAdaGradSGDRulelearning_rateinitial_g2sumg      @initial_rangeg-C6?g      $g      $@&SparseSharedAdamSGDRulebeta1_decay_rateg?beta2_decay_rateg+?ada_epsilong:0yE>))r!   r>   r?   r@   HasFieldrC   rB   rE   r   graph_sgd_paramr   r   ctr_accessor_paramr   r   r   r   r   r   r   r   r   r   embed_sgd_paramembedx_sgd_paramadagradr   r   r   lenweight_boundsextendnaiveinitializerssplitfloatr   attrsr   r   r   )rQ   rF   r2   r   r   common_accessorr3   r4   r5   rG   rH   r   r   	sgd_paramr   r   x	attr_lists                     r    ra   Accessor._set   sR    .?.
*s ))+Cxx7" #		! ,
 &&'788|$0A-0@-&&y11,,0@@)6):&)6&&&|44,,0@@,9A,=),9A,=)&&'9::./N+(88''66*.O'''(?@@48O1+>>!**;77+/($$)+0(!**>::.1+!**=99-0*!**+;<<01-!**+<==12.!**+<==13.!**+BCC784!**+=>>23/!**+EFF:<7!**+DEE9:6 **++
I %%f--"11U:%9IN"11V;%8IN%8IN "88>>%88 ((11/BB6:I%%3 ((11/BB69I%%3 ((11/BB6<I%%3y((6671<%%33::E4=I~~!55 //@@$3$@$@$D$J$J3$O%M 5:-4HIOO1 //@@$3$@$@$C$I$I#$N%M 5:-4HIOO1y445:OO1188%G "55>>%>> ~~..??$3$@$@$D$J$J3$O%M 493GINN0 ~~..??$3$@$@$C$I$I#$N%M 493GINN03B3H3HI3HaQWWS\3H	I!//0BCC'66&@6;IaLO6LINN369INN3!//0BCC'66&@6;IaLO6LINN36;INN3!//>>'66&@16y|A1GINN.16INN.y~~3349NN0077F[
l Js   /^)rC   rG   r   r   Nrc   rO   r"   r    r{   r{   }   s    JGr"   r{   c                   L   ^  \ rS rSrU 4S jrS rS rS rS rS r	S r
S	rU =r$ )
CommonAccessori  c                    > [         TU ]  5         SU l        SU l        / U l        / U l        / U l        SU l        SU l        / U l	        0 U l
        0 U l        0 U l        U R                  5         g )Nr~   r(   r   F)rn   rR   
table_nameentryr   paramsdimstrainer_numsyncr   opt_input_mapopt_attr_mapopt_init_mapdefine_optimize_mapro   s    r    rR   CommonAccessor.__init__  sl    

		  "r"   c                    0 nSS/US'   / SQUS'   / SQUS'   S/US'   / S	QUS
'   SS/US'   0 n/ US'   / US'   / US
'   / SQUS'   / SQUS'   S/US'   0 n/ SQUS'   S/US'   / SQUS'   / SQUS'   X l         Xl        X0l        g )NParamNLearningRater:   r   )r   )Moment1N)Moment2N)Beta1Powr:   )Beta2Powr:   r   r   )r   )D2SumN)G2SumN)MomentN)MomentDecayRater:   )AdaDecayRater:   )
AdaEpsilonr:   r   
adam_d2sumsum)r   )r   r:   r   naive_adagrad)SummaryDecayRater:   summary))beta1f)beta2r   )epsilonr   )summary_decay_rater   )seedmeanstdgaussian_randomvaluefill_constant)r   minmaxuniform_randomtruncated_gaussian_random)r   r   r   )rQ   r   r   r   s       r    r   "CommonAccessor.define_optimize_map"  s     /1DEe!
f	'
l# !00e*
o&
 %45L#Mi  U U(*_% 
V
&
\"
 $?"?Y*A&')0	_%)?%&4K01(*(r"   c                 h   [        X25      u  pEnUR                  5       R                   H  n[        U5      (       d  [	        U5      (       d  M%  UR                  S5      S   nX:X  a(  UR                  S:X  a  UR                  S5      U l          g X:X  d  Mm  UR                  S:X  d  M  SU l          g    g )Nr$   r   r%   r   r&   r(   )	r!   r*   r+   r,   r-   r.   r/   r1   r   )	rQ   r2   r   r   r3   r4   r5   r6   r7   s	            r    parse_entryCommonAccessor.parse_entryY  s    ->.
*s ++-11B+B//R8H8H#q)J$N)BWWW-
$4E)E#
 2r"   c                 \    [        X-  S-   5      nXCS-   -  U::  a  U$ XC-  U:  a  XU-  -
  $ g)Nr:   r   )int)rQ   	total_dim	shard_num
pserver_id	blocksizes        r    	get_shardCommonAccessor.get_shardk  sE    	-12	Q'94%	1 z#999r"   c           	         SnSnUnUR                  5       R                   H  nUR                  U R                  R	                  5       ;   d  M-  XVR                  S5      S   :X  d  MF  UR                  /nU R                  UR                      H,  nUR                  [        UR                  U5      5      5        M.     UR                  U5      n  U$    U$ )Nr   r~   Outr   )
r*   r+   r/   r   keysoutputappendstrr1   join)	rQ   
value_nameo_startup_programl_inattr_strorigin_var_namer6   	init_attrr1   s	            r    get_initializer_attr#CommonAccessor.get_initializer_attrv  s    $#00266B4,,1133#yy'7'::WWI	 --bgg6D$$S%78 7  99Y/ 7 r"   c           	         UR                  5       S   nUR                  5       nUR                  5       S   nUR                  5       (       a  UR                  5       S   OSnUS   R                  n[	        X!R                  5       5      u  pn
[        US   5      n[        [        US   5      5      n[        U5      nS nU H7  nSUR                  ;   d  M  UR                  S5      S   US   U   :X  d  M5  Un  O   Uc  [        SU 35      e/ n/ n/ n/ n[        US   5      U l        XPl        X`l        UR"                  S:w  a  U(       a  [%        S	5        S
n[%        SU5        US   [&        R(                  :X  a'  U R*                  S   nU R,                  S   nSU l        GOUS   (       a-  U(       a&  U R*                  S   nU R,                  S   nSU l        OUR1                  5       (       a&  U R*                  S   nU R,                  S   nSU l        OU(       a-  U(       d&  U R*                  S   nU R,                  S   nSU l        OnUR"                  S:w  a  UR"                  S:w  a  [        S5      eU R*                  UR"                     nU R,                  UR"                     nUR"                  U l        U GH  u  nnUR3                  U5        U R.                  S:X  Ga  Uc  U(       a  UnOU R5                  X\U5      nUR3                  U5        US:X  d  US:X  a  UR7                  5       R8                  UR                  U5      S      nUS:X  a[  UR:                  S[=        U
5      -   :w  a?  [>        R@                  " S5        UR7                  5       R8                  S[=        U
5      -      nU RC                  UR:                  U	5      nOUS:X  a  SnOUS:X  a  SnOUS:X  a  SnOSnUR3                  U5        GMC  U R.                  S:X  a  Uc  U(       a  UnOU R5                  X\U5      nUR3                  U5        US:X  aL  UR7                  5       R8                  UR                  U5      S      nU RC                  UR:                  U	5      nOUS:X  a  SnOSnUR3                  U5        GM  US :X  a'  UR3                  S5        SnUR3                  U5        GM!  UR7                  5       R8                  UR                  U5      S      nUS:X  a[  UR:                  S[=        U
5      -   :w  a?  [>        R@                  " S5        UR7                  5       R8                  S[=        U
5      -      nUc  U(       a  UnOU R5                  X\U5      nUR3                  U5        U RC                  UR:                  U	5      nUR3                  U5        GM     U R.                  S:X  aH  [E        U5      nU H7  nS!UR                  ;   d  M  UR                  S!5      S   US   U   :X  d  M5  Un  O   U HB  u  nnURG                  U5      nUR3                  S"RI                  U[=        U5      /5      5        MD     UU l%        UU l&        UU l'        UU l(        g )#Nr   r:   user_defined_strategy
role_makerr   grad_name_to_param_namezcan not find optimizer for r   z8optimization algorithm is not adam, set adam_d2sum FalseFzadam_d2sum:ps_moder   r   r   r   r   r   z8The dense optimizer in PS is only supported SGD or Adam!r   learning_rate_zwill support decay soonr   zfill_constant&0.99r   zfill_constant&0.9999r   zfill_constant&1.0e-8zfill_constant&0r   zfill_constant&0.999999r   	BatchSizer   ))origin_varnames	is_sparsesectionsr   r!   r   get_role_idr   get_ps_endpointsget_optimize_opsinput_namesr.   rD   get_trainersr   	table_num	table_dimr/   rA   DistributedModeGEOr   r   rC   is_datanorm_tabler   r   r*   varsr?   r   warningswarnr   get_datanorm_opsr1   r   r   r   r   r   )rQ   ctxr   	grad_namer
  size
single_dimr   r3   r4   r5   r   pserver_numoptimizer_opsoopr6   r   r   r   r   param_varnamesattr_varnamesformal_namer@   paraminitializerdatanorm_opsattr_varnametype_r   s                                 r    parse_by_optimizer!CommonAccessor.parse_by_optimizer  s*   '')!,	MMO	||~a *---//S\\^A&q
45@@
 .?^^%.
*s !!67
*7<+@AB(6 B2>>)!!$45i@A    ;:9+FGG'(=>#88v*LMJmZ(9!4!44!//6N --e4M"'D\"y!//@N --o>M"'D""$$!//	:N --i8M"+D	!//=N --l;M".Dxx5 SXX%7 N  "//9N --chh7M"%((D"0KMM+&""l2=  * $t* ME" ')[N-J(557<<		+.q1E
 $~5!JJ*:SX*EE &?@ , 9 9 ; @ @,s3x7! #'";";

O#K !$55"6K N2"8K L0"8K"3K##K0$$	1=  * $t* ME" ')(557<<		+.q1E #'";";

O#K !$66":K"3K##K0')KKN"3K ''4(557<<		+.q1E $~5!JJ*:SX*EE &?@ , 9 9 ; @ @,s3x7! }$$.E$(NN $:%E KK&"&";";

O#K !''4w #1z )++L9L"2>>1HH[)!,89)DE C # $1L%HH\*ELL<U"<=> $1 	(
r"   c                    U R                   Ul        U R                  Ul        UR                  R	                  U R                  5        UR
                  R	                  U R
                  5        UR                  R	                  U R                  5        U R                  Ul        U R                  Ul        U R                  Ul	        U R                  Ul
        U R                  Ul        SR                  U R                  5      Ul        g )N#)rC   r?   r   r   r   r   r   r   r   r   r  r  r   r   r1   rQ   protos     r    ra   CommonAccessor._set@  s    ((
??DKK(

$))$!!$"3"34jj ,,YY
....XXdjj)
r"   )rC   r   r   r   r   r   r   r   r   r   r  r   r  r   )rd   re   rf   rg   rR   r   r   r   r   r)  ra   rh   rx   ry   s   @r    r   r     s.    #5)n$	*rj* *r"   r   c                        \ rS rSrS rS rSrg)TensoriN  c                     Xl         g rN   tensor_dict)rQ   r4  s     r    rR   Tensor.__init__O  s    &r"   c                 N   U R                   R                  SS5      Ul        U R                   R                  SS5      Ul        U R                   R                  SS5      Ul        U R                   R                  SS5      Ul        U R                   R                  SS5      Ul        g )Nmain_program_idr   startup_program_idfeed_var_namer~   fetch_var_nametensor_table_class)r4  getr7  r8  r9  r:  r;  )rQ   tensor_protos     r    ra   Tensor._setR  s    '+'7'7';';q(
$ +/*:*:*>*> !+
' &*%5%5%9%9/2%N"&*&6&6&:&:;KR&P#*.*:*:*>*> "+
'r"   r3  Nrc   rO   r"   r    r1  r1  N  s    '
r"   r1  c                        \ rS rSrS rS rSrg)r   i`  c                     S U l         SU l        S U l        [        5       U l        SU l        [        5       U l        S U l        g )Nr      )r'   r   r/   r{   accessorr   commontensorrP   s    r    rR   Table.__init__a  s9    	 
$&r"   c                     g rN   rO   )rQ   table_protos     r    ra   
Table._setj  rT   r"   )rB  rC  r   r'   rD  r/   Nrc   rO   r"   r    r   r   `  s    r"   r   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )r   in  c                   > [         TU ]  5         S U l        SU l        SU R                  l        SU R                  l        / U R                  l        / U R                  l	        US   U l
        US   U l        X l        US   U l        g )NrA  CommMergeAccessorr~   is_heter_ps_moder  is_sync)rn   rR   r/   r   rB  rC   rC  r   r   r   rL  r  r5   rM  )rQ   r   r5   rp   s      r    rR   BarrierTable.__init__o  sw    	':$ '(: ;!,/y)r"   c                 &   U R                   Ul        SUl        SUl        [        R
                  Ul        SUR                  l        SUR                  l	        SUR                  l
        SUR                  l        SUR                  l        U R                  UR                  l        SUR                  l        [#        U R$                  5      nU R&                  (       a&  U[)        U R$                  R+                  5       5      -  nX!R                  l        g )Nr   rA  rK  r   r~   barrier_tabler(   )r5   table_idr'   r   r   PS_OTHER_TABLEr/   rB  rC   rB   rE   rC  r?   r   rM  r   r   r  r  rL  r   _get_heter_worker_endpointsr   )rQ   rG  r   s      r    ra   BarrierTable._set|  s    #xx"0 #)88.A+'($*+'"$(7%"&,,#) "4??3  3tJJLMMK)4&r"   )r5   rL  rM  r  r   r/   rw   ry   s   @r    r   r   n  s    *5 5r"   r   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )r   i  c                 F   > [         TU ]  5         Xl        X l        X0l        g rN   )rn   rR   r5   r4  r  )rQ   r5   r4  r  rp   s       r    rR   TensorTable.__init__  s    &$r"   c                    U R                   Ul        [        R                  Ul        U R
                  R                  SS5      Ul        SUR                  l	        U R
                  R                  SS5      UR                  l        [        U R                  5      UR                  l        [        U R
                  5      nUR!                  UR"                  5        g )Nr;  r~   rK  r9  )r5   rQ  r   rR  r/   r4  r<  r'   rB  rC   rC  r   r  r  r   r1  ra   rD  )rQ   rG  rD  s      r    ra   TensorTable._set  s    #xx)88"&"2"2"6"67KR"P.A+(,(8(8(<(<R)
% *6doo)F&(()K&&'r"   )r5   r  r4  rw   ry   s   @r    r   r     s    %( (r"   r   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )r   i  c                 t   > [         TU ]  5         Xl        X l        S U l        SU l        [        5       U l        g )Nr)   )rn   rR   r   r  r/   r'   r{   rB  rQ   r   send_ctxrp   s      r    rR   SparseTable.__init__  s0    	. 
r"   c                 
   U R                   nUR                  5       (       d2  [        UR                  5       5      S:  d  UR	                  5       (       d  g UR                  5       Ul        U R                  Ul        [        R                  Ul	        U R                  Ul
        UR                  [        [        U R                  S   5      5      :  a&  [        [        U R                  S   5      5      Ul        U R                  S   UR                  5       S      U R                  l        U R                  R!                  X R                  5        U R                  R#                  U R                  R                  UR%                  5       U R                  5        U R                  S   (       a  SOSU R                  l        U R                  R)                  UR                  5        [+        SU R                  R                   35        U R                  S	   R,                  nUR/                  5       nU H+  nUR                  U R                  R                  :X  d  M)  Un  O   UR1                  S
5      (       a  UR                  Ul        OSUl        [2        R4                  " S5        UR1                  S5      (       a  UR                  Ul
        OOU R                  S   (       a  SUl
        [2        R4                  " S5        OSUl
        [2        R4                  " S5        UR1                  S5      (       a  UR6                  Ul        UR1                  S5      (       a  UR8                  Ul        UR1                  S5      (       a  UR                  Ul        UR1                  S5      (       a  UR:                  Ul        UR1                  S5      (       a  UR<                  Ul        UR>                  RA                  5       S:X  a  [2        R4                  " S5        UR1                  S5      (       a  URB                  Ul!        UR>                  RE                  UR>                  RG                  5       5        U R>                  R)                  UR>                  U R                  R                  UR%                  5       U R                  U R                  5        [I        UR>                  U R                  R                  UR%                  5       U R                  5        g )Nr:   r  r  r   rM  TFznew table_name: r  r'   r)   z'The PS mode must use MemorySparseTable.r   r   %   zHThe shard_num of sparse table is not set, use default value 37 in gpups.i  zJThe shard_num of sparse table is not set, use default value 1000 in cpups.enable_sparse_table_cachesparse_table_cache_ratesparse_table_cache_file_numenable_revertshard_merge_ratez;The accessor of sparse table is not set, use default value.use_gpu_graph)%r  is_tensor_tabler   r	  r
  rQ  r'   r   PS_SPARSE_TABLEr/   r   rc  r  r   rC  r   r)  r   r   r   ra   rA   sparse_table_configsaddr   r  r  ra  rb  rd  re  rB  ByteSizerf  ParseFromStringSerializeToStringrI   )rQ   rG  r  all_table_protousr_table_protor.  s         r    ra   SparseTable._set  s   hh!!3&&()A-MMOO"||~"&"2"2)99 $22ST\\,786
 
 7: l!;<7K3 "&.G!H!!$"
 	&&sLL9KK""CNN$4dll	
 $(<<	#:4++, !7!7 89:,,#


 	 *--/$E4;;#9#99"' % ##M22&5&A&AK#&9K#MMCD##K00$3$=$=K!||L)(*%^ )-%` ##$?@@99 1 ##$=>>77 / ##$ABB;; 3 ##O44(7(E(EK%##$677+:+K+KK(##,,.!3MMM ##O44(7(E(EK%,,$$668	
 	  KK""NNLLKK	
 	  KK""NNLL		
r"   )rB  r   r  r'   r/   rw   ry   s   @r    r   r     s    #c
 c
r"   r   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )r   i  c                    > [         TU ]  X5        SU l        U R                  S   [        R
                  :w  a  [        S5      eg )NMemorySparseGeoTabler  znot geo sparse table!)rn   rR   r'   r   r  r  rD   r\  s      r    rR   GeoSparseTable.__init__  s@    +1<<	"o&9&99455 :r"   c                    U R                   nUR                  5       (       d2  [        UR                  5       5      S:  d  UR	                  5       (       d  g UR                  5       Ul        U R                  Ul        [        R                  Ul	        U R                  Ul
        SUR                  l        UR                  5       S   UR                  l        UR                  5       S   UR                  l        U R                   S   UR                  5       S      U R"                  l        U R"                  R'                  X R                   5        U R"                  R)                  U R"                  R$                  UR+                  5       U R                   5        SU R"                  l        U R"                  R/                  UR"                  5        g )Nr:   rK  r   r  F)r  rg  r   r	  r
  rQ  r'   r   rh  r/   r   rB  rC   r  rB   rE   r   rC  r   r)  r   r   r   ra   rQ   rG  r  s      r    ra   GeoSparseTable._set  sL   hh!!3&&()A-MMOO"||~"&"2"2)99 $.A+'*||~a'8$*-,,.*;'!%.G!H!!$"
 	&&sLL9KK""CNN$4dll	
 !++,r"   )r'   rw   ry   s   @r    r   r     s    6- -r"   r   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )r   i:  c                 X   > [         TU ]  5         Xl        X l        [	        5       U l        g rN   )rn   rR   r   r  r{   rB  r\  s      r    rR   DenseTable.__init__;  s!     
r"   c                 :   U R                   nUR                  5       (       d2  [        UR                  5       5      S:  d  UR	                  5       (       a  g UR                  5       Ul        [        R                  Ul        SUl	        SUl
        SUR                  l        UR                  5       S   UR                  l        SUR                  l        SU R                   l        U R                   R%                  X R&                  5        U R                   R)                  U R                   R"                  UR+                  5       U R&                  5        U R&                  S   (       a  SOS	U R                   l        U R                   R/                  UR                   5        g )
Nr:   MemoryDenseTablerA  rK  r   MergedDenserM  TF)r  rg  r   r	  r
  rQ  r   PS_DENSE_TABLEr/   r'   r   rB  rC   r  rB   rE   rC  r   r)  r   r   r   r   ra   rv  s      r    ra   DenseTable._setA  s"   hh!!3&&()A-"||~)88"4 #.A+'*||~a'8$*+'!.&&sLL9KK""CNN$4dll	
 $(<<	#:4++,r"   )rB  r   r  rw   ry   s   @r    r   r   :  s    #- -r"   r   c                        \ rS rSrS rS rSrg)Serveri^  c                     g rN   rO   rP   s    r    rR   Server.__init___  rT   r"   c                     g rN   rO   rP   s    r    ra   Server._setb  rT   r"   rO   Nrc   rO   r"   r    r  r  ^      r"   r  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )DownpourServerif  c                 "   > [         TU ]  5         g rN   rm   ro   s    r    rR   DownpourServer.__init__g  rr   r"   c                     g rN   rO   rP   s    r    ra   DownpourServer._setj  rT   r"   rO   rw   ry   s   @r    r  r  f       r"   r  c                        \ rS rSrS rS rSrg)Workerin  c                     g rN   rO   rP   s    r    rR   Worker.__init__o  rT   r"   c                     g rN   rO   rP   s    r    ra   Worker._setr  rT   r"   rO   Nrc   rO   r"   r    r  r  n  r  r"   r  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )DownpourWorkeriv  c                 "   > [         TU ]  5         g rN   rm   ro   s    r    rR   DownpourWorker.__init__w  rr   r"   c                     g rN   rO   rP   s    r    ra   DownpourWorker._setz  rT   r"   rO   rw   ry   s   @r    r  r  v  r  r"   r  c                        \ rS rSrS rS rSrg)fsClienti~  c                     Xl         g rN   fs_client_param)rQ   r  s     r    rR   fsClient.__init__  s    .r"   c                 (   [         R                  " U R                  5      (       d  g U R                  R                  Ul        U R                  R                  Ul        U R                  R
                  Ul        U R                  R                  Ul        g rN   )r   MessageToStringr  uriuserpasswd
hadoop_binr-  s     r    ra   fsClient._set  si    **4+?+?@@((,,	))..
++22//::r"   r  Nrc   rO   r"   r    r  r  ~  s    /;r"   r  c                   D    \ rS rSrS rS rS rS rS rS r	S r
S	 rS
rg)PsDescBuilderi  c                    Xl         US   U l        US   U l        US   U l        US   U l        S U l        [        U R                   U R                  S9U l        0 U l        / U l	        U R                  5       U l        U R                  5       U l        U R                  5       U l        [         R"                  " 5       U l        [         R&                  " 5       U l        g )NrM  r  rL  r   split_dense_table)r   rM  r  rL  r   barrier_table_idget_the_one_send_contextr]  tensor_table_dict_server_sub_program_get_tablestables_get_serviceservice_get_fs_client	fs_clientr   PSParameterps_descFLParameterfl_desc)rQ   r   s     r    rR   PsDescBuilder.__init__  s    y)y) '(: ;!,/ $0LLD,A,A
 "$#% &&(((*,,.%113%113r"   c           	      6   SnU R                   (       d-  U R                  R                  [        5       R                  5        / nU R                    HF  nUR                  [        5       S   " [        U5      [        U R                  S   5      5        US-  nMH     U$ )Nr   r   r  r:   )	r  r  r   r   descglobalsr   r4  r   )rQ   program_idxr  r   s       r    _get_tensor_tables PsDescBuilder._get_tensor_tables  s    %%$$++GINN;00JMM	-(Kdll<.H
 1K 1 r"   c                    / n[        U R                  R                  5       5       GHV  u  nu  p4[        SX#U5        UR	                  5       (       a  U R
                  [        R                  :X  a  U R                  S   (       a  US S U R                  S   ;   d  U R                  S   (       d/  UR                  [        5       S   " U R                  U5      5        M  UR                  [        5       S   " U R                  U5      5        M  UR                  [        5       S   " U R                  U5      5        GM  U R                  (       a  GM)  UR                  [        5       S   " U R                  U5      5        GMY     U R                  5       U l        UR                  U R                  5        UR                  [        5       S   " U R                  [        U5      5      5        U$ )Nzidx, name, ctx:local_sparser   r   r   r   )r   r]  itemsrA   r
  r  r  r  r   r   r  r   r  tensor_tablesr   r   )rQ   r  r5   r?   r  s        r    r  PsDescBuilder._get_tables  s\    )$--*=*=*? @C$#S4}}<<?#6#66^4 "In)EE"ll>:#I&67cJ #Im4T\\3G MM')M":4<<"MNMM')L"9$,,"LM' !A( "446d(()gi/c&kJKr"   c                 L    U R                   (       a
  [        5       $ [        5       $ rN   )r   rj   rK   rP   s    r    r  PsDescBuilder._get_service  s    ??<9r"   c                 F    [        U R                  S   R                  5      $ )Nr  )r  r   r  rP   s    r    r  PsDescBuilder._get_fs_client  s    %<=MMNNr"   c                     g rN   rO   )rQ   client_infos     r    build_fl_client_desc"PsDescBuilder.build_fl_client_desc  rT   r"   c                    U R                    H  nU R                  R                  R                  R                  R                  5       nUR                  U5        U R                  R                  R                  R                  R                  5       nUR                  U5        [        U5      [        :X  d  M  U R                  b  M  UR                  U l        M     U R                  R                  U R                  R                  R                  R                  5        U R                  R                  U R                  R                   5        ["        R$                  " U R                  5      $ rN   )r  r  worker_paramdownpour_worker_paramdownpour_table_paramrj  ra   server_paramdownpour_server_paramr/   r   r  r5   r  service_paramr  r  r   r  rQ   tablerG  s      r    build_worker_descPsDescBuilder.build_worker_desc  s    [[E,,33II^^bbdKJJ{#,,33II^^bbdKJJ{#E{l*t/D/D/L(-		% ! 	LL%%;;II	
 	DLL889**4<<88r"   c                    0 U l         U R                   H  nU R                  R                  R                  R
                  R                  5       nUR                  U5        UR                  [        R                  :X  d  Ml  UR                  c  M{  UR                  U R                   UR                  R                  '   M     U R                  R                  U R                  R                  R                  R                  5        U R                   R                  U R                  R"                  5        [$        R&                  " U R                  5      $ rN   )sparse_table_mapsr  r  r  r  r  rj  ra   r/   r   rh  rC  rQ  r   r  r  r  r  r   r  r  s      r    build_server_descPsDescBuilder.build_server_desc  s    !#[[E,,33II^^bbdKJJ{#  N$B$BB&&2  (( &&{'9'9'D'DE ! 	LL%%;;II	
 	DLL889**4<<88r"   )r  r  r   r  r  rL  rM  r  r  r]  r  r  r  r  r  r   N)rd   re   rf   rg   rR   r  r  r  r  r  r  r  rh   rO   r"   r    r  r    s+    4.6O99r"   r  c                   $  ^  \ rS rSrU 4S jrS rS rS rS rS r	S$S jr
S$S	 jrS
 rS%S jrS rS r\/ 4S j5       rS r S$S jrS r S&S jr   S'S jrS rS rS rS r S(S jrS rS rS rS rS r S r!S r"S  r#S! r$S$S" jr%S#r&U =r'$ ))TheOnePSRuntimei  c                    > [         TU ]  5         S U l        S U l        [        R
                  " 5       U l        S U l        / U l        S U l	        S U l
        g rN   )rn   rR   _communicator_serverr   DistFleetWrapper_worker_coordinatorr  _heter_client	_send_ctxro   s    r    rR   TheOnePSRuntime.__init__  sL    !,,. #% !r"   c                    Xl         US   U l        [        U R                  5      U l        [	        [        [        R                  " SS5      5      5      U l        US   U l	        UR                  SU R                  /5      U l        U R                  U R                   S'   UR                  SUS   /5      U R                   S'   U R                  R                  U R                   S'   U R                   S   U l        [        US	   5      U R                   S
'   U R                   S
   R                  U R                   S'   US	   R                   S   U R                   S'   US	   R                   S   U R                   S'   U R                   S   ["        R$                  :X  a  SOSU R                   S'   0 U R                   S'   0 U R                   S'   US   R&                  S   U R                   S'   US   R&                  S   U R                   S'   [)        SR+                  U R                   S   U R                   S   5      5        [-        U R                   5        [/        U R                  5      U l        [3        U R                  5      U l        / U l        [9        U R4                  5       Hb  u  p#UR;                  S5      u  pE[<        R>                  " U[        U5      U5      nU R6                  RA                  URC                  5       5        Md     U R                  RD                  U l#        / U l$        U RF                  (       a  [)        SU R6                   35        U R                  RK                  5       n[9        U5       Hb  u  p#UR;                  S5      u  p[<        R>                  " U[        U5      U5      nU RH                  RA                  URC                  5       5        Md     [M        U R                   5      U l'        g )Nr  PSERVER_DEBUG0origin_main_programr   r   origin_startup_programrL  valid_strategytrainerr  r   rf  TFrM  r  tensor_tabler  r  remote_sparsez+fl-ps > local_sparse: {}, remote_sparse: {}:zfl-ps > all ps addrs: )(r   r  r  role_idboolr   osgetenvdebugr  r<  r   _is_heter_parameter_server_moderL  TrainerRuntimeConfigmodea_sync_configsr  SYNCtrainer_desc_configsrA   formatbuild_var_distributedget_trainer_endpointstrainer_endpointsr  	endpointsstring_hostsr   r   r   PSHostr   serialize_to_string_with_coordinatorwith_coordinatorcoordinator_hosts_get_coordinator_endpointsr  ps_desc_builder)	rQ   r   r5   ephostportpshostcoordinator_endpointsips	            r    _set_basic_infoTheOnePSRuntime._set_basic_info  sa   !,/"4??3#bii=>?
#*+@#A $+KK"T%=%=$>%
! 04/H/H+,29++%0H(I'J3
./ OO;; 	'( !%-? @"6$%#
Y #',,y"9">">Y%,-=%>%M%M&
\" )0)

.)*_% LL+/C/CCD 	Y 35./')^$'.#(


~(/^$ )0#)


)0_% 	9@@^,dll?.K	
 	dll+!6t!G)$//: 0GC#JD[[s4y#6F$$V%?%?%AB 1
 !% A A!#  *4+<+<*=>?$(OO$N$N$P!$%:;88C=RTC8&&--f.H.H.JK <
  -T\\:r"   c                 B   / nUR                  5        H  u  pVUR                  5       (       a  M  [        U R                  UR	                  5       5      u    pxX   n	UR                  5       n
X:   nU R                  R                  XU5        UR                  U5        M     U$ rN   )	r  r
  r!   r   r   rQ  r  push_dense_paramsr   rQ   scopesr]  recv_mapall_var_namesr?   r  _r5   scoperQ  	var_namess               r    _init_all_params TheOnePSRuntime._init_all_paramsJ      !)ID}})$,,8HIIAqKE||~H *ILL**5IF  + * r"   c                 B   / nUR                  5        H  u  pVUR                  5       (       a  M  [        U R                  UR	                  5       5      u    pxX   n	UR                  5       n
X:   nU R                  R                  XU5        UR                  U5        M     U$ rN   )	r  r
  r!   r   r   rQ  r  pull_dense_paramsr   r  s               r    _pull_all_denseTheOnePSRuntime._pull_all_denseX  r  r"   c                 *   / nUR                  5        H|  u  pgUR                  5       (       a  M  UR                  5       [        U5      :w  a  M;  UR	                  5       nXH   n	U R
                  R                  X(U	5        UR                  U	5        M~     U$ rN   )r  r
  r   r   rQ  r  r  r   
rQ   r   r  r]  r  r  r?   r  rQ  r  s
             r    _init_paramsTheOnePSRuntime._init_paramsf  {    !)ID}}~~2g;.||~H *ILL**5IF  + * r"   c                 *   / nUR                  5        H|  u  pgUR                  5       (       a  M  UR                  5       [        U5      :w  a  M;  UR	                  5       nXH   n	U R
                  R                  X(U	5        UR                  U	5        M~     U$ rN   )r  r
  r   r   rQ  r  r   r   r$  s
             r    _pull_denseTheOnePSRuntime._pull_denset  r'  r"   c                   ^  T R                   R                  5       n/ n[        T R                  S   [        5      (       az  [        T R                  S   5      S:  a^  [        [        T R                  S   5      5       H8  nUR                  T R                  S   U   R                  R                  5        M:     O2UR                  T R                  S   R                  R                  5        [        [        U5      5       H  nT R                  S   (       a~  X4   R                  (       d	  0 X4   l
        SX4   R                  S'   [        R                  " S5      nUR                  S5       Vs/ s H  n[        U5      PM     nnXSU   R                  S'   T R                  S   (       d  M  SX4   R                  S'   M     U 4S	 jn[        T R                  T R                   S
9n[#        T R                  T R                   T R$                  S9n	U	T l        T R                  S   n
T R(                  (       aO  [+        SU 35        [+        S5        U	 H  n[+        U SX    35        M     U H  n[+        U SX    35        M     0 nSUS'   T R,                  R/                  5       US'   T R,                  R1                  5       US'   T R                   R2                  US'   T R                  S   [4        R6                  :X  a  U" 5       nUR9                  U5        [+        SU
R;                  5       5        T R<                  R?                  UT R@                  T RB                  5        T R                   (       d2  [E        T R,                  5      T l#        [+        ST RF                   35        [+        ST RH                   35        [+        ST RJ                   35        T RH                  (       a;  T R<                  RM                  T RJ                  T RB                  T RF                  5        T R                  S   [4        RN                  :X  d  T R                   (       ao  [Q        U
RR                  UU
R;                  5       5      T l*        T RT                  RW                  U	UUT R@                  [X        RZ                  R]                  5       5        [^        R`                  Rc                  5         T R<                  Re                  5       n[        U[        5      (       a  [        U5      S:  a  T R,                  Rg                  US   5      n[        U[        5      (       d  [h        Rj                  " S5        U/nT R<                  Rm                  U5        T R<                  Ro                  5         [+        S5        O[+        S5        T R                  S   n[q        [        [        R                  " SS5      5      5      nUcC  [        T Rr                  5      S:  a  [u        S 5      e[X        RZ                  R]                  5       /n[        T Rr                  5      [        U5      :w  a  [u        S!5      eUT l;        U(       d  T R                  S   [4        RN                  :X  d  T R                   (       a  T RT                  Ry                  U5        OAT R                  S   (       d-  T RB                  S:X  a  [+        S"5        T R{                  XU5        [^        R`                  Rc                  5         T R                  S   (       d  T R}                  XU5        [^        R`                  Rc                  5         T R                  S   [4        RN                  :X  d  T R                   (       aP  T RT                  R                  5       (       d  T RT                  R                  5         O[h        Rj                  " S#5        UR                  S$   n[        [        R                  " S%S&5      5      nU(       Ga/  U(       Ga&  [        T R,                  R                  5       5        T R                   (       aA  T R,                  R                  5       / :w  a#  [        T R,                  R                  5       5        T R                   (       a  / nT R,                  R                  5       / :w  a  T R,                  R                  5       n/ nT R,                  R                  5       / :w  a  T R,                  R                  5       n[        UUT R,                  R/                  5       5      T lG        g g g g s  snf )'Nlossr:   r   TFLAGS_selected_gpus,worker_placesrf  c                     > 0 n TR                   R                  5       U S'   TR                   R                  5       U S'   U $ )Npserver_endpoints
trainer_id)r  _get_pserver_endpoints_worker_index)kwargsrQ   s    r    sync_strategy_envs8TheOnePSRuntime._init_worker.<locals>.sync_strategy_envs  sA    F668 &' $(??#@#@#BF< Mr"   r  r  ep_listr  zworker_desc: 
zcommunicator send_ctx:z: r  need_global_stepr2  trainersr  r  zcommunicator config:zfl-ps > trainer_endpoint: zfl-ps > with_coordinator? zfl-ps > coordinator addr: r   z!gloo may not initialize correctlyzcreate c2c connection donezcannot create c2c connectionr  	TEST_MODEz;You must set the scope list when you have Multiple programszlen(programs) != len(scopes)z entering self._init_all_params()z'communicator has been initialized, skiplaunch_barrierFLAGS_LAUNCH_BARRIER1)Hr
  r  
isinstancer   listr   ranger   blockr   
_fleet_optr  r  r   r   get_the_one_recv_contextrL  r  r  r  r  rA   r  _role_id_worker_numr  r  r  updateget_communicator_flagsr  init_workerr  r  get_trainer_endpointtrainer_endpointr  r  init_fl_workerr  r   r  r  init_with_ctxpaddlestaticglobal_scoper   utilbarrierget_client_info_all_gatherr  r  set_clientscreate_client2client_connectionr  r   rD   r  init_paramsr  r!  
is_runningstartr  r   r3  _get_next_trainers_get_previous_trainersr   r  )rQ   r  worker_descmain_programsr   gpus_envsr6  	dense_mapr]  trainer_configkeyr5  sync_kwargsinfoall_infodist_strategyis_testr=  launch_barrier_flagprevious_trainersnext_trainerss   `                     r    _init_workerTheOnePSRuntime._init_worker  s,   **<<>t||F+T22DLL()A-3t||F345$$T\\&%9!%<%B%B%J%JK 6   f!5!;!;!C!CDs=)*A||L)$'2224M$/<@ ++L999%:;,4NN3,?@,?qCF,?@?Ga ++O<||O,,?C ++O< +	 -LLD,A,A
	 ,LL"33NN

 "i0::OK=12*+R/0   R	/01 ! %(!"#779|!__88:z%)%9%9%J%J!"<<	"o&:&::,.KMM+&$n&K&K&MN  d.?.?N$$$8$ID!.t/D/D.EFG*4+@+@*ABC*4+A+A*BCD  LL''&&d6K6K
 LL#':'::$$!-##557"D
 ,,!!**, 	

 ||++-dD!!c$i!m22QH h--AB$: LL$$X.LL88:./01%56s299[#678>4,,-1 Q  mm0023Ft(()S[8;<<Y'?+>+>>((""..y9||L1||q(@A--f	JJJ ||L)  9=

 LL#':'::$$%%0022""((*GH&556FG!")),BC"HI1dooDDFG%%OO668B>!$//"D"D"FG$$$&!??99;rA(,(N(N(P% "??5572=$(OO$F$F$HM%0!#4doo6N6N6P&" % 2>Q As   -ec                    U R                   c  [        U R                  5      U l         [        SU R                  S    35        [        SU R
                   35        U R                   R                  U R                  S   U R
                  5        g )Nz>>> curr node ip: r   z>>> all trainer endpoints: )r  r
   r  rA   r  r  start_coordinator)rQ   r  s     r    _init_coordinator!TheOnePSRuntime._init_coordinator.  s~    $ +D,=,= >D"4#9#9!#<"=>?+D,B,B+CDE++""1%t'='=	
r"   c                 T    U R                   c  g U R                   R                  5         g rN   )r  make_fl_strategyrP   s    r    _make_fl_strategy!TheOnePSRuntime._make_fl_strategy8  s#    $3..0r"   c                 l   U R                   R                  5       n[        U R                  5      nU R                  (       a&  U[        U R                  R                  5       5      -  nU R                  (       a  [        SU 35        [        R                  " 5       U l        U R                  R                  UU R                  U R                  UU R                  5        [!        U R"                  S5      n[!        U R"                  S5      nXg-   nUc  Un	OU H  n
X;  d  M
  [%        SU 35      e   Un	Ub  U	(       d  g U R                   R&                  n[(        R*                  R-                  U5      nU R                  R/                  5       nU	 H$  n
X   nU R                  R1                  USU5        M&     g )Nzserver_desc: 
TFz4fleet.init server can only load sparse variables in r  )r
  r  r  r  rL  r   rS  r  rA   r   r  r  init_serverr  r  r  get_sparse_tablenamesr   rD   r  r  pathnormpathrF  load_sparse)rQ   dirnamer  r5  server_descr;  dist_varnamessparse_varnamesdistributed_varnamesload_varnamesvar_namer  r   rQ  s                 r    _init_serverTheOnePSRuntime._init_server>  sy   **<<>0  DOOGGIJJH::OK=12,,.  LL$$	
 .d.G.GN/%%u
  ->0M%7$NOcNde  &
 &M?- 00BB''""7+__--/
%H(2HLL$$Wc8< &r"   c                     [        U R                  5      nUR                  S5      u  p#U R                  R	                  U[        U5      5        g )Nr  )get_ps_endpointr  r   r  
run_serverr   )rQ   r  r  r  s       r    _run_serverTheOnePSRuntime._run_serverm  s7    T__-XXc]
c$i0r"   c                 .   U R                   S   [        R                  :X  a  U R                  R	                  5         U R
                  R                  5         U R                  (       a/  U R                  c   S5       eU R                  R	                  5         g g )Nr  z/heter client should not be None in heterps mode)	r   r  r  r  stopr  stop_workerrL  r  rP   s    r    _stop_workerTheOnePSRuntime._stop_workerr  sy    <<	"o&9&99##%  "  %%1 A1 ##%	 !r"   c                    ^  U 4S jnU$ )Nc                 T  > U R                   T;   a  gSSKJn  U" U R                   5      u  n  nUR                  S5      (       a  gUR	                  S5      (       a  gU R
                  R                  5       [        R                  R                  R                  :X  d  U R
                  R                  5       [        R                  R                  R                  :X  d@  U R
                  R                  5       [        R                  R                  R                  :X  a  gU R                  $ )NFr:   )_get_varname_partsz@GRADr  )r?   utils.publicr  endswith
startswithr  r/   r   VarDescVarTypeFEED_MINIBATCH
FETCH_LISTREADERpersistable)rH   r  origin_varnamer  exclude_var_namess       r    is_valid0TheOnePSRuntime.__exclude_vars.<locals>.is_valid~  s    xx,,8#5chh#? NAq&&w//(()9:: 4<<#7#7#F#FF88==?dll&:&:&E&EE88==?dll&:&:&A&AA??"r"   rO   )r  r  s   ` r    __exclude_varsTheOnePSRuntime.__exclude_vars|  s    	#* r"   c                     UR                  S5      (       d  UR                  S5      (       a  SnU$ [        R                  R                  US5      nU$ )Nzafs:zhdfs:z./dnn_plugin
dnn_plugin)r  r  ry  r   )rQ   r|  
model_paths      r    _get_inference_model_path)TheOnePSRuntime._get_inference_model_path  sL    f%%););G)D)D'J  g|<Jr"   c           	      *   [        U R                  U R                  S9n[        U R                  U R                  U R                  S9nUb  [        U R                  5      S:X  a  U R                  S   nU R                  XCXv5      nUc  UOUn	U	 V
s/ s H!  oR                  5       R                  U
5      PM#     nn
SS K
nUR                  R                  U5         UR                  R                  USXKUS9  S S S 5        g s  sn
f ! , (       d  f       g = f)Nr  r8  r:   r   z./)r  filename)rE  r   rL  r  r  r   r   r)  r*   rH   rO  rP  scope_guard	save_vars)rQ   executorr|  r  r   r  ra  r]  dense_var_namessave_var_namesr   r  rO  s                r    _ps_save_dense_params%TheOnePSRuntime._ps_save_dense_params  s     -LLD,A,A
	 ,LL"33NN

 ?c$";";<A//2G**78O,5,=97EF~!$$&**1-~F]]&&u-MM##$W $  .- G .-s   (C?D
Dc                 <   [        U R                  S5      n/ nU R                  U5      nUR                  5        HW  u  pU
S   U;  a   U R                  R                  X5        U R                  R                  XU5        UR                  U
5        MY     U$ !    N7= fNTr   )rx  r   r  r  r  recv_and_save_modelsave_one_modelr   )rQ   r  r|  r   r3   r  r  valuesr  r   namess              r    _save_sparse_params#TheOnePSRuntime._save_sparse_params  s      5%%t 
 33G<
 IBQx33LL44RD LL''T:MM%  ) s   	BBc                     [        U[        5      (       d  [        S5      eUc  U R                  S   n[        U[        5      (       a  [        S5      eU R
                  R                  X$5        g)a  
This function filters out all variables with `persistable==True` from the
give `main_program` and then saves these variables to the folder `dirname`
or file `filename`.

The `dirname` is used to specify the folder where persistable variables
are going to be saved. If you would like to save variables in separate
files, set `filename` None; if you would like to save all variables in a
single file, use `filename` to specify the file name.
;in fleet.save() function, executor must be as Executor typeNr  ^in fleet.save() function, main_program must be as Program type, CompiledProgram is not allowed)r@  r   	TypeErrorr   r   r  save_all_model)rQ   r  r|  r3   r  r5  s         r    _save_distributed_persistables.TheOnePSRuntime._save_distributed_persistables  si     (H--M  <<(=>LlO44p  	##G2r"   c                    [        U[        5      (       d  [        S5      eSSKnUc  U R                  S   OUn	[        U R                  [        U	5      5      u    pU R                  U   n[        SU5        [        U	[        5      (       a  [        S5      eU Vs/ s H!  oR                  5       R                  U5      PM#     nnUR                  R                  XU5      nUR                  U	5        U R!                  U5      nSn["        R$                  R'                  UU5      nUR(                  " UU5        [+        U R                  SU R,                  S9nU R/                  XUXW5      n[+        U R                  U R,                  S	9n[1        U R                  U R,                  U R2                  S
9nU R5                  XUU5        U R                  S   R6                  S   n[9        U5      n[9        [;        [<        R?                  U5      URA                  5       5      5      nU HN  nURC                  U5      nUR(                  " U["        R$                  R'                  UURD                  5      SS9  MP     gs  snf )z
Prune the given `main_program` to build a new program especially for inference,
and then save it and all related parameters to given `dirname` by the `executor`.
r  r   Nzsave inference model scope idx:r  	__model__Fis_denser  r  r8  r  stat_var_namesT)use_binary_format)#r@  r   r  rO  r   r!   r   r   r  rA   r   r*   rH   rP  normalize_program_copy_dist_param_info_fromr  r  ry  r   saverE  rL  r  r  r  r)  r  rA  filterr  _TheOnePSRuntime__exclude_varsr>   	get_valuer?   )rQ   r  r|  feeded_var_namestarget_varsr3   export_for_deploymentr  rO  r   r  r5   r  r?   	feed_varsinfer_programr  model_basenamesparsessparse_namesra  r]  generate_varsremaining_varsrH   rD  s                             r    "_ps_inference_save_inference_model2TheOnePSRuntime._ps_inference_save_inference_model  sQ    (H--M  	 # %%a( 	
 &dllBwK@	1C /5g//p 
 :J
9I  "&&t,9I 	 
 77
 	00933G<
$j.AM>2*LL"33

 //w
 -LLD,A,A
	 ,LL"33NN

 	9=#


/1 ]+..|<'')
 "C]]5)FKKZ2"& "[
s   (I5c                    UR                  SS5      nUR                  SS5      nU R                  R                  5         [        R                  R                  5         SnU R                  R                  5       (       a  U R                  R                  U5      n[        R                  R                  5         U R                  R                  5       (       a  U R                  R                  XAX55        [        R                  R                  5         SnU R                  R                  5       (       a  U R                  R                  XAU5      n[        R                  R                  5         U$ )Nr  r:   rQ  r   g        r   )r<  r  client_flushr   rR  rS  r  _is_first_workerget_cache_thresholdcache_shuffle
save_cache)rQ   r|  r5  r  rQ  cache_thresholdfeasign_nums          r    _save_cache_model!TheOnePSRuntime._save_cache_model>  s    zz&!$::j!,!!#

??++--"ll>>xHO

??++--LL&&x$P

??++--,,11(TJK

r"   c                    [         R                  R                  5         U R                  S   (       d  U R                  R                  5       (       a  U R                  R                  XU5        [         R                  R                  5         g Nr   )r   rR  rS  r   r  r  r  save_cache_table)rQ   rQ  pass_idmem_cache_key_thresholds       r    _save_cache_table!TheOnePSRuntime._save_cache_tableV  sZ    

<<%)I)I)K)KLL))#: 	

r"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R
                  R                  5         g g rN   )r   rR  rS  r  r  r  check_save_pre_patch_donerP   s    r    _check_save_pre_patch_done*TheOnePSRuntime._check_save_pre_patch_done^  s:    

??++--LL224 .r"   c                     [        U R                  S5      n/ nUR                  5        HQ  u  pxUS   U;  a  [        R                  " S5        U R
                  R                  XqU5        UR                  U5        MS     U$ )NTr   z,varname is not in distributed_varnames, pass)rx  r   r  r  r  r  load_one_tabler   )	rQ   r|  r   r3   r  r  r  r   r  s	            r    _load_sparse_params#TheOnePSRuntime._load_sparse_paramsc  sr    4%%t 
  IBQx33LMLL''T:MM%  ) r"   c                    Uc  U R                   S   OUn[        U R                  [        U5      5      u    pEU R                  U   n[        SU5        [        U[        5      (       a  [        S5      e[        U R                  SU R                  S9nU R                  XX25      n[        U R                  U R                  S9n	[        U R                  U R                  U R                  S9n
/ nU	R                  5        H  u  pLUR                  U5        M     Un[!        [#        [$        R'                  U5      UR)                  5       5      5      nU R+                  U5      nSS KnU Ha  nUR.                  U;  a  M  UR0                  " [2        R4                  R7                  UUR.                  5      5      nUR9                  UU5        Mc     U R;                  X6X5        g )Nr   zload inference model scope idx:r  Fr  r  r8  )r   r!   r   r   r  rA   r@  r   r  rE  rL  r  r  r  r  r   rA  r  r  r  r>   r  rO  r?   loadr  ry  r   	set_valuer%  )rQ   r|  r  r3   r  r5   r  r  r  ra  r]  recv_dense_varnamesr  loaded_varnamesr  r  rO  rH   rD  s                      r    "_ps_inference_load_inference_model2TheOnePSRuntime._ps_inference_load_inference_modelq  s   
 # %%a( 	
 &dllB|4DE	1C /5lO44p  +LL"33
 22l
 -LLD,A,A
	 ,LL"33NN
 !!)HA&&u- * *..?&&(
 33G<
!Cxx22[[j#((!CDFMM&%(	 " 	,xCr"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R
                  R                  XU5        [         R                  R                  5         g rN   )r   rR  rS  r  r  r  r  rQ   rQ  ry  r  s       r    _save_one_tableTheOnePSRuntime._save_one_table  I    

??++--LL''=

r"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R
                  " U0 UD6  [         R                  R                  5         g rN   )r   rR  rS  r  r  r  rQ   argsr5  s      r    _save_dense_params"TheOnePSRuntime._save_dense_params  sH    

??++--&&77

r"   c                    [         R                  R                  5         U R                  S   (       d  U R                  R                  5       (       a  U R                  " U0 UD6  [         R                  R                  5         g r  )r   rR  rS  r   r  r  r  r  s      r    _save_persistables"TheOnePSRuntime._save_persistables  sS    

<<%)I)I)K)K//@@

r"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R
                  " U0 UD6  [         R                  R                  5         g rN   )r   rR  rS  r  r  r  r  s      r    _save_inference_model%TheOnePSRuntime._save_inference_model  sH    

??++--33TDVD

r"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R
                  R                  XU5        [         R                  R                  5         g rN   )r   rR  rS  r  r  r  r  r  s       r    _load_one_tableTheOnePSRuntime._load_one_table  r  r"   c                    [         R                  R                  5         U R                  S   (       d  U R                  R                  5       (       a  U R                  R                  X5        [         R                  R                  5         g r  )r   rR  rS  r   r  r  r  
load_modelrQ   ry  r  s      r    _load_persistables"TheOnePSRuntime._load_persistables  sR    

<<%)I)I)K)KLL##D/

r"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R                  X5        [         R                  R                  5         g rN   )r   rR  rS  r  r  r  r  s      r    _load_inference_model%TheOnePSRuntime._load_inference_model  sC    

??++--33D?

r"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R
                  R                  X5        [         R                  R                  5         g rN   )r   rR  rS  r  r  r  set_date)rQ   rQ  day_ids      r    	_set_dateTheOnePSRuntime._set_date  sG    

??++--LL!!(3

r"   c                     [         R                  R                  5         U R                  R	                  5       (       a  U R
                  R                  XU5        [         R                  R                  5         g rN   )r   rR  rS  r  r  r  print_table_stat)rQ   rQ  r  	thresholds       r    _print_table_stat!TheOnePSRuntime._print_table_stat  sI    

??++--LL))(YG

r"   c                    Ub  [         R                  " S5        OSn[        R                  R	                  5         U R
                  S   (       d  U R                  R                  5       (       a]  [        U R
                  SU R                  R                  S9nUR                  5        H   u  p4U R                  R                  X15        M"     [        R                  R	                  5         g )NznThe param threshold is not used in MemorySparseTable, if you need to shrink, please set the config of accessorr   r   Fr  )r  r  r   rR  rS  r   r  r  rE  r  r  r  shrink_sparse_table)rQ   r  r  r   r  s        r    _shrinkTheOnePSRuntime._shrink  s     MM A I

<<%)I)I)K)K."&//"Q"QG %]]_	00? -

r"   )r  r  r  r  r  r  r  r   r  r  r  rL  r  r   r
  r  r  r  r  rL  r  r  rN   )NN)Nr   r  )r   N)(rd   re   rf   rg   rR   r  r  r!  r%  r)  rl  rp  rt  r  r  r  staticmethodr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rh   rx   ry   s   @r    r  r    s    C;JjX
1-=^1
& )+  0 <@., :;3F "Wr05
 -1:Dx r"   r  )/r  r  google.protobufr   rO  paddle.distributedr   paddle.distributed.communicatorr   r   5paddle.distributed.fleet.base.private_helper_functionr   paddle.distributed.fleet.protor   -paddle.distributed.fleet.runtime.runtime_baser	   !paddle.distributed.ps.coordinatorr
   "paddle.distributed.ps.utils.publicpaddle.frameworkr   paddle.staticr   r   r   __all__r!   r8   rI   rK   rj   r{   r   r1  r   r   r   r   r   r   r  r  r  r  r  r  r  rO   r"   r    <module>r+     s'   
  '  $ E : E 9 0 ! < <+&D	- 	-5 5RG RGjy*X y*x	
 
$  55  5F(% (.l
% l
^ -[  -F!- !-H V  V 
; 
;l9 l9^{k {r"   