
    x-j                       d dl Z d dlZd dlmZ d dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lT d d
lmZ d dlmZmZmZ g dZd Zd Zd Z G d d          Z G d de          Z G d d          Z G d de          Z  G d d          Z! G d d          Z" G d de"          Z# G d de"          Z$ G d  d!e"          Z% G d" d#e%          Z& G d$ d%e"          Z' G d& d'          Z( G d( d)e(          Z) G d* d+          Z* G d, d-e*          Z+ G d. d/          Z, G d0 d1          Z- G d2 d3e          Z.dS )4    N)text_format)fleet)CommunicatorHeterClient)wait_server_ready)the_one_ps_pb2)RuntimeBase)Coordinator)*)core)CompiledProgramExecutorProgram)TableSparseTableGeoSparseTableBarrierTableTensorTable
DenseTablec                     | d         }t          |          D ]+\  }}t          |          |k    r|| d         |         |fc S ,dS )Norigin_main_programsorigin_startup_programs)NNN)	enumerateid)context
program_idprogramsiprograms        `/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/ps/the_one_ps.pyget_program_by_idr!   +   sd    -.H)) E E
7g;;*$$G$=>qA1DDDD %    c                    t          ||          \  }}}|                                j        D ]}t          |          st	          |          s!|                    d          d         }|| k    r|j        dk    s|j        dk    rH|                    d          r0|                    d          dk    r|                    d          c S  dS d S )NWr   lookup_tablelookup_table_v2table_classnoneMemorySparseTable)	r!   global_blockopsis_distributed_sparse_opis_sparse_opinputtypehas_attrattr)varnamer   r   main_programstartup_programidxop
param_names           r    parse_table_classr8   3   s    ):7J)O)O&L/3''))- + +'++ 	L4D4D 	XXc]]1%
 '!!>))w+++{{=)) +bggm.D.D.N.Nww}-----*** ,+ +r"   c           	         t          ||          \  }}}d}|                                D ]9}|j        |k    r,|j        d         }t	          d| d| d| j                     n:| j        }	| j        dk    r"|	|dz   k    rt          d|dz    d|	           n|	|k    rt          d	| d|	           | j        }
| j        dk    r#|
|dz
  k    rt          d
|dz
   d|
           d S |
|dz
  k    rt          d|dz
   d|
           d S )Nr      z	new var: z, SparseAccessor   z;The fea_dim is wrong, it will be sparse_embedding_dim + 2: z
, but got z7The fea_dim is wrong, it will be sparse_embedding_dim: z>The embedx_dim is wrong, it will be sparse_embedding_dim - 1:    z>The embedx_dim is wrong, it will be sparse_embedding_dim - 3: )	r!   	list_varsnameshapeprintfea_dimaccessor_class
ValueError
embedx_dim)accessor_protor2   r   r   r3   r4   r5   embedding_dimvarrB   rE   s              r    check_embedding_dimrI   F   s   ):7J)O)O&L/3M%%''  8wIaLMNcNN]NNn6LNNOOOE 
 $G$(888ma'''tm^_N_ttkrtt   (
 m##l-llcjll    *J$(888***zQ^abQbzznxzz   +*
 ***zQ^abQbzznxzz   +*r"   c                       e Zd Zd Zd ZdS )Servicec                     d S N selfs    r    __init__zService.__init__i       r"   c                 L    d|_         d|_        d|_        d|_        d|_        d S )NBrpcPsServerBrpcPsClientBrpcPsServicer      )server_classclient_classservice_classstart_server_portserver_thread_numrP   service_protos     r    _setzService._setl   s0    %3"%3"&5#*+'*,'''r"   N__name__
__module____qualname__rQ   r_   rN   r"   r    rK   rK   h   s2          - - - - -r"   rK   c                   $     e Zd Z fdZd Z xZS )
GpuServicec                 H    t                                                       d S rM   superrQ   rP   	__class__s    r    rQ   zGpuService.__init__u       r"   c                 "    d|_         d|_        d S )NPsLocalServerPsLocalClient)rX   rY   r]   s     r    r_   zGpuService._setx   s    %4"%4"""r"   ra   rb   rc   rQ   r_   __classcell__rj   s   @r    re   re   t   sG            5 5 5 5 5 5 5r"   re   c                       e Zd Zd Zd ZdS )Accessorc                 >    d| _         d | _        d| _        d| _        d S )N r   )rC   	optimizerfeature_dimrG   rO   s    r    rQ   zAccessor.__init__~   s'     r"   c                 6   t          ||          \  }}}d}	|                                D ]}
|
j        |k    r|
j        d         }	 n|                    d          s|d         rd|_        nd|_        |                    d          s|j        dk    r|	dz   |_        n|	|_        |                    d	          s |j        dk    r|	dz
  |_        n
|	d
z
  |_        |                    d          sd|_        |j	        }|                    d          sd|_
        |                    d          sd|_        |j        }|                    d          sd|_        |j        dk    rd|_        |                    d          sd|_        |                    d          sd|_        |                    d          sd|_        |                    d          sd|_        |                    d          sd|_        |                    d          sd|_        |                    d          sd|_        |                    d          sd|_        |                    d          sd|_        |j        |j        fD ]"}|                    d           s,|j        d!k    rd"|_        |j        d#k    rd$|_        nd$|_        |j        d%k    s|j        d&k    r|j                            d'          sd|j        _        |j                            d(          sd)|j        _        |j                            d*          sd+|j        _        t;          |j        j                  dk    r!|j        j                            d,d-g           |j        d"k    r|j                             d'          s?|j!        d.         "                    d/          d         }tG          |          |j         _        |j                             d*          s?|j!        d         "                    d/          d.         }tG          |          |j         _        t;          |j         j                  dk    r!|j         j                            d,d-g           |j        d$k    s|j        d0k    r|j$                            d'          s?|j!        d.         "                    d/          d         }tG          |          |j$        _        |j$                            d*          s?|j!        d         "                    d/          d.         }tG          |          |j$        _        d1 |j%        D             }|j$                            d2          s1|j        d#k    r&tG          |d         d                   |j$        _&        nd3|j$        _&        |j$                            d4          s1|j        d#k    r&tG          |d         d                   |j$        _'        nd5|j$        _'        |j$                            d6          s1|j        d#k    r&tG          |d         d                   |j$        _(        nd7|j$        _(        t;          |j$        j                  dk    r!|j$        j                            d,d-g           $d S )8Nr   r:   rC   
use_ps_gpuCtrDymfAccessorr;   rB   r<   rE   r=   embedx_thresholdnodeid_sloti0#  feature_learning_rateg?	zero_initTFnonclk_coeffg?click_coeffg      ?base_thresholddelta_thresholddelta_keep_days   show_click_decay_ratedelete_thresholddelete_after_unseen_days   ssd_unseenday_thresholdr?   sgdSparseNaiveSGDRuleadamSparseAdamSGDRuleSparseAdaGradSGDRuleStdAdaGradSGDRulelearning_rateinitial_g2sumg      @initial_rangeg-C6?g      $g      $@&SparseSharedAdamSGDRulec                 8    g | ]}|                     d           S )r   )split).0xs     r    
<listcomp>z!Accessor._set.<locals>.<listcomp>   s"    IIIaQWWS\\IIIr"   beta1_decay_rateg?beta2_decay_rateg+?ada_epsilong:0yE>))r!   r>   r?   r@   HasFieldrC   rB   rE   r{   graph_sgd_paramr|   r}   ctr_accessor_paramr~   r   r   r   r   r   r   r   r   r   embed_sgd_paramembedx_sgd_paramadagradr   r   r   lenweight_boundsextendnaiveinitializersr   floatr   attrsr   r   r   )rP   rF   r2   r   r   common_accessorr3   r4   r5   rG   rH   r   r   	sgd_paramr   r   	attr_lists                    r    r_   zAccessor._set   s    .?Z.
 .
*os ))++ 	 	Cx7"" #	! # &&'788 	A|$ A0A--0@-&&y11 	7,0@@@)6):&&)6&&&|44 	>,0@@@,9A,=)),9A,=)&&'9:: 	0./N+(8''66 	/*.O'''(?@@ 	948O1+>!**;77 	0+/($))+0(!**>:: 	2.1+!**=99 	1-0*!**+;<< 	201-!**+<== 	312.!**+<== 	413.!**+BCC 	9784!**+=>> 	423/!**+EFF 	=:<7!**+DEE 	;9:6 *+
 M	G M	GI %%f-- 9"1U::%9IN"1V;;%8INN%8IN "888>%888 (11/BB ;6:I%3 (11/BB :69I%3 (11/BB =6<I%3y(6771<<%3::E4=III~!555 //@@ I$3$@$D$J$J3$O$O%M 5:-4H4HIO1 //@@ I$3$@$C$I$I#$N$N%M 5:-4H4HIO1y455::O188%GGG "555>%>>> ~..?? H$3$@$D$J$J3$O$O%M 493G3GIN0 ~..?? H$3$@$C$I$I#$N$N%M 493G3GIN0II?3HIII	!//0BCC:'6&@@6;IaLO6L6LIN3369IN3!//0BCC<'6&@@6;IaLO6L6LIN336;IN3!//>>7'6&@@16y|A1G1GIN..16IN.y~34499N077FFF[M	G M	Gr"   Nr`   rN   r"   r    rs   rs   }   s<          JG JG JG JG JGr"   rs   c                   B     e Zd Z fdZd Zd Zd Zd Zd Zd Z	 xZ
S )CommonAccessorc                 
   t                                                       d| _        d| _        g | _        g | _        g | _        d| _        d| _        g | _	        i | _
        i | _        i | _        |                                  d S )Nru   r(   r   F)rh   rQ   
table_nameentryr   paramsdimstrainer_numsyncr   opt_input_mapopt_attr_mapopt_init_mapdefine_optimize_mapri   s    r    rQ   zCommonAccessor.__init__  s    

		  """""r"   c                 
   i }ddg|d<   g d|d<   g d|d<   dg|d<   g d	|d
<   ddg|d<   i }g |d<   g |d<   g |d
<   g d|d<   g d|d<   dg|d<   i }g d|d<   dg|d<   g d|d<   g d|d<   || _         || _        || _        d S )NParamNLearningRater:   r   )r   )Moment1N)Moment2N)Beta1Powr:   )Beta2Powr:   r   r   )r   )D2SumN)G2SumN)MomentN)MomentDecayRater:   )AdaDecayRater:   )
AdaEpsilonr:   r   
adam_d2sumsum)r   )r   r:   r   naive_adagrad)SummaryDecayRater:   summary))beta1f)beta2r   )epsilonr   )summary_decay_rater   )seedmeanstdgaussian_randomvaluefill_constant)r   minmaxuniform_randomtruncated_gaussian_random)r   r   r   )rP   r   r   r   s       r    r   z"CommonAccessor.define_optimize_map"  s:    /1DEe!
 !
 !
f	'
 	'
 	'
l# !00e*
 *
 *
o&
 %45L#Mi  U U(*_% 
  
  
V
&
 &
 &
\"
 $?"?Y*A*A*A&')0	_%)?)?)?%&4K4K4K01(*(r"   c                 l   t          ||          \  }}}|                                j        D ]}t          |          st	          |          s!|                    d          d         }||k    r(|j        dk    r|                    d          | _         d S ||k    r|j        dk    r
d| _         d S d S )Nr$   r   r%   r   r&   r(   )	r!   r*   r+   r,   r-   r.   r/   r1   r   )	rP   r2   r   r   r3   r4   r5   r6   r7   s	            r    parse_entryzCommonAccessor.parse_entryY  s    ->Z.
 .
*os ++--1 	 	B+B// R8H8H #q)JW$$N)B)BWWW--
W$$4E)E)E#
	 	r"   c                 n    t          ||z  dz             }||dz   z  |k    r|S ||z  |k     r|||z  z
  S dS )Nr:   r   int)rP   	total_dim	shard_num
pserver_id	blocksizes        r    	get_shardzCommonAccessor.get_shardk  sX    	I-122	
Q'944:%	11 9z#999qr"   c           	         d}d}|}|                                 j        D ]}|j        | j                                        v r||                    d          d         k    ri|j        g}| j        |j                 D ]7}|                    t          |                    |                               8|	                    |          } n|S )Nr   ru   Outr   )
r*   r+   r/   r   keysoutputappendstrr1   join)	rP   
value_nameo_startup_programl_inattr_strorigin_var_namer6   	init_attrr1   s	            r    get_initializer_attrz#CommonAccessor.get_initializer_attrv  s    $#00226 	 	B4,113333#ryy'7'7':::WI	 -bg6 9 9D$$S%7%7888899Y//r"   c           	         |                                 d         }|                                }|                                d         }|                                r|                                d         nd}|d         j        }t	          ||                                          \  }}	}
t          |d                   }t          t          |d                             }t          |          }d }|D ]:}d|j
        v r/|                    d          d         |d         |         k    r|} n;|t          d|           g }g }g }g }t          |d                   | _        || _        || _        |j        dk    r|rt%          d	           d
}t%          d|           |d         t&          j        k    r"| j        d         }| j        d         }d| _        n|d         r$|r"| j        d         }| j        d         }d| _        n|                                r"| j        d         }| j        d         }d| _        n{|r$|s"| j        d         }| j        d         }d| _        nU|j        dk    r|j        dk    rt          d          | j        |j                 }| j        |j                 }|j        | _        |D ]?\  }}|                    |           | j        dk    r+||r|}n|                     |||          }|                    |           |dk    s|dk    r|                                j        |                    |          d                  }|dk    r^|j        dt=          |
          z   k    rCt?          j         d           |                                j        dt=          |
          z            }| !                    |j        |	          }n|dk    rd}n|dk    rd}n|dk    rd}nd}|                    |           R| j        dk    r||r|}n|                     |||          }|                    |           |dk    rT|                                j        |                    |          d                  }| !                    |j        |	          }n|dk    rd}nd}|                    |           |d k    r.|                    d           d}|                    |           @|                                j        |                    |          d                  }|dk    r^|j        dt=          |
          z   k    rCt?          j         d           |                                j        dt=          |
          z            }||r|}n|                     |||          }|                    |           | !                    |j        |	          }|                    |           A| j        dk    rLtE          |          }|D ]:}d!|j
        v r/|                    d!          d         |d         |         k    r|} n;|D ]Q\  }}|#                    |          }|                    d"$                    |t=          |          g                     R|| _%        || _&        || _'        || _(        d S )#Nr   r:   user_defined_strategy
role_makerr   grad_name_to_param_namezcan not find optimizer for r   z8optimization algorithm is not adam, set adam_d2sum FalseFzadam_d2sum:ps_moder   ry   r   r   r   r   z8The dense optimizer in PS is only supported SGD or Adam!r   learning_rate_zwill support decay soonr   zfill_constant&0.99r   zfill_constant&0.9999r   zfill_constant&1.0e-8zfill_constant&0r   zfill_constant&0.999999r   	BatchSizer   ))origin_varnames	is_sparsesectionsr   r!   r   get_role_idr   get_ps_endpointsget_optimize_opsinput_namesr.   rD   get_trainersr   	table_num	table_dimr/   rA   DistributedModeGEOr   r   rC   is_datanorm_tabler   r   r*   varsr?   r   warningswarnr   get_datanorm_opsr1   r   r   r   r   r   )rP   ctxr   	grad_namer   size
single_dimr   r3   r4   r5   r   pserver_numoptimizer_opsoopr6   r   r   r   r   param_varnamesattr_varnamesformal_namer@   paraminitializerdatanorm_opsattr_varnametype_r   s                                 r    parse_by_optimizerz!CommonAccessor.parse_by_optimizer  se   ''))!,	MMOO	||~~a *---//@S\\^^A&&q
45@
 .?S^^%%.
 .
*os !!677
*7<+@AABB(66  	 	B2>))!!!$45i@A A ;F9FFGGG'(=>>#8v*LMMMJmZ(((9!444!/6N -e4M"'D\" 	+y 	+!/@N -o>M"'D""$$ 	+!/	:N -i8M"+D 	+	 	+!/=N -l;M".Dx5  SX%7%7 N   "/9N -ch7M"%(D"0 [	5 [	5KMM+&&&"l22=  N * $t[* M ME""" '))[N-J-J(5577<		+..q1E
 $~55!J*:SXX*EEE &?@@@ , 9 9 ; ; @,s3xx7! #'";";
O# #KK !$555"6KK N22"8KK L00"8KK"3K##K0000$	11=  N * $t[* M ME""" '))(5577<		+..q1E #'";";
O# #KK !$666":KK"3K##K0000'))KKNNN"3K ''4444(5577<		+..q1E $~55!J*:SXX*EEE &?@@@ , 9 9 ; ; @,s3xx7! }$ $.EE$(NN $k:% %E KK&&&"&";";
O# #K !''4444)+++L99L"  2>11HH[))!,89)DE E CE#0 	? 	?L%HH\**ELL<U"<==>>>>	(


r"   c                    | j         |_        | j        |_        |j                            | j                   |j                            | j                   |j                            | j                   | j        |_        | j        |_        | j	        |_	        | j
        |_
        | j        |_        d                    | j                  |_        d S )N#)rC   r?   r   r   r   r   r   r   r   r   r  r  r   r   r1   rP   protos     r    r_   zCommonAccessor._set@  s    (
?DK(((
$)$$$!!$"3444j ,Y
..XXdj))


r"   )ra   rb   rc   rQ   r   r   r   r   r  r_   rp   rq   s   @r    r   r     s        # # # # #5) 5) 5)n  $	 	 	  *r r rj* * * * * * *r"   r   c                       e Zd Zd Zd ZdS )Tensorc                     || _         d S rM   )tensor_dict)rP   r%  s     r    rQ   zTensor.__init__O  s    &r"   c                 F   | j                             dd          |_        | j                             dd          |_        | j                             dd          |_        | j                             dd          |_        | j                             dd          |_        d S )Nmain_program_idr   startup_program_idfeed_var_nameru   fetch_var_nametensor_table_class)r%  getr'  r(  r)  r*  r+  )rP   tensor_protos     r    r_   zTensor._setR  s    '+'7';';q(
 (
$ +/*:*>*> !+
 +
' &*%5%9%9/2%N%N"&*&6&:&:;KR&P&P#*.*:*>*> "+
 +
'''r"   Nr`   rN   r"   r    r#  r#  N  s2        ' ' '
 
 
 
 
r"   r#  c                       e Zd Zd Zd ZdS )r   c                     d | _         d| _        d | _        t                      | _        d| _        t                      | _        d | _        d S )Nr      )r'   r   r/   rs   accessorr   commontensorrO   s    r    rQ   zTable.__init__a  sB    	 

$&&r"   c                     d S rM   rN   )rP   table_protos     r    r_   z
Table._setj  rR   r"   Nr`   rN   r"   r    r   r   `  s2              r"   r   c                   $     e Zd Z fdZd Z xZS )r   c                     t                                                       d | _        d| _        d| j        _        d| j        _        g | j        _        g | j        _	        |d         | _
        |d         | _        || _        |d         | _        d S )Nr0  CommMergeAccessorru   is_heter_ps_moder   is_sync)rh   rQ   r/   r   r1  rC   r2  r   r   r   r9  r   r5   r:  )rP   r   r5   rj   s      r    rQ   zBarrierTable.__init__o  s    	':$ '(: ;!,/y)r"   c                    | j         |_        d|_        d|_        t          j        |_        d|j        _        d|j        _	        d|j        _
        d|j        _        d|j        _        | j        |j        _        d|j        _        t#          | j                  }| j        r)|t)          | j                                                  z  }||j        _        d S )Nr   r0  r8  r   ru   barrier_tabler(   )r5   table_idr'   r   r   PS_OTHER_TABLEr/   r1  rC   rB   rE   r2  r?   r   r:  r   r   r  r   r9  r   _get_heter_worker_endpointsr   )rP   r5  r   s      r    r_   zBarrierTable._set|  s    #x"0 #)8.A+'($*+'"$(7%"&,#) "4?33  	N3tJJLLMMMK)4&&&r"   ro   rq   s   @r    r   r   n  sG        * * * * *5 5 5 5 5 5 5r"   r   c                   $     e Zd Z fdZd Z xZS )r   c                 r    t                                                       || _        || _        || _        d S rM   )rh   rQ   r5   r%  r   )rP   r5   r%  r   rj   s       r    rQ   zTensorTable.__init__  s3    &$r"   c                 z   | j         |_        t          j        |_        | j                            dd          |_        d|j        _	        | j                            dd          |j
        _        t          | j                  |j
        _        t          | j                  }|                    |j                   d S )Nr+  ru   r8  r)  )r5   r=  r   r>  r/   r%  r,  r'   r1  rC   r2  r   r  r   r   r#  r_   r3  )rP   r5  r3  s      r    r_   zTensorTable._set  s    #x)8"&"2"6"67KR"P"P.A+(,(8(<(<R)
 )
% *6do)F)F&())K&'''''r"   ro   rq   s   @r    r   r     sG        % % % % %( ( ( ( ( ( (r"   r   c                   $     e Zd Z fdZd Z xZS )r   c                     t                                                       || _        || _        d | _        d| _        t                      | _        d S )Nr)   )rh   rQ   r   r  r/   r'   rs   r1  rP   r   send_ctxrj   s      r    rQ   zSparseTable.__init__  sE    	. 

r"   c                    | j         }|                                s9t          |                                          dk     s|                                sd S |                                |_        | j        |_        t          j        |_	        | j
        |_
        |j        t          t          | j        d                             k    r,t          t          | j        d                             |_        | j        d         |                                d                  | j        _        | j                            || j                   | j                            | j        j        |                                | j                   | j        d         rdnd| j        _        | j                            |j                   t+          d| j        j                    | j        d	         j        }|                                }|D ]}|j        | j        j        k    r|} n|                    d
          r|j        |_        nd|_        t3          j        d           |                    d          r|j
        |_
        nD| j        d         rd|_
        t3          j        d           nd|_
        t3          j        d           |                    d          r|j        |_        |                    d          r|j        |_        |                    d          r|j        |_        |                    d          r|j        |_        |                    d          r|j        |_        |j                                         dk    rt3          j        d           |                    d          r|j!        |_!        |j        "                    |j        #                                           | j                            |j        | j        j        |                                | j        | j                   tI          |j        | j        j        |                                | j                   d S )Nr:   r   r   r   r:  TFznew table_name: r   r'   r)   z'The PS mode must use MemorySparseTable.r   ry   %   zHThe shard_num of sparse table is not set, use default value 37 in gpups.i  zJThe shard_num of sparse table is not set, use default value 1000 in cpups.enable_sparse_table_cachesparse_table_cache_ratesparse_table_cache_file_numenable_revertshard_merge_ratez;The accessor of sparse table is not set, use default value.use_gpu_graph)%r  is_tensor_tabler   r   r   r=  r'   r   PS_SPARSE_TABLEr/   r   rK  r  r   r2  r   r  r   r   r   r_   rA   sparse_table_configsaddr   r  r  rI  rJ  rL  rM  r1  ByteSizerN  ParseFromStringSerializeToStringrI   )rP   r5  r  all_table_protousr_table_protor!  s         r    r_   zSparseTable._set  sq   h!!	3&&(())A--MMOO . F"||~~"&"2)9 $2ST\,7886
 6
 
 
 7: l!;<<7 7K3 "&.G!H!!!$"
 	&&sDL999K"CNN$4$4dl	
 	
 	
 $(<	#:E44+,,,9!799:::,#

 	 *--//$ 	 	E4;#999"' : ##M22 	E&5&AK##&9K#MCDDD##K00 	$3$=K!!|L) 	(*%^    )-%`   ##$?@@ 	9 1 ##$=>> 	7 / ##$ABB 	; 3 ##O44 	F(7(EK%##$677 	L+:+KK(#,,..!33MM   ##O44 	F(7(EK%,,$6688	
 	
 	
 	 K"NNLK	
 	
 	
 	 K"NNL		
 	
 	
 	
 	
r"   ro   rq   s   @r    r   r     sN        # # # # #c
 c
 c
 c
 c
 c
 c
r"   r   c                   $     e Zd Z fdZd Z xZS )r   c                     t                                          ||           d| _        | j        d         t          j        k    rt          d          d S )NMemorySparseGeoTabler   znot geo sparse table!)rh   rQ   r'   r   r  r  rD   rE  s      r    rQ   zGeoSparseTable.__init__  sQ    (+++1<	"o&9994555 :9r"   c                 X   | j         }|                                s9t          |                                          dk     s|                                sd S |                                |_        | j        |_        t          j        |_	        | j
        |_
        d|j        _        |                                d         |j        _        |                                d         |j        _        | j        d         |                                d                  | j        _        | j                            || j                   | j                            | j        j        |                                | j                   d| j        _        | j                            |j                   d S )Nr:   r8  r   r   F)r  rO  r   r   r   r=  r'   r   rP  r/   r   r1  rC   r   rB   rE   r   r2  r   r  r   r   r   r_   rP   r5  r  s      r    r_   zGeoSparseTable._set  sb   h!!	3&&(())A--MMOO . F"||~~"&"2)9 $.A+'*||~~a'8$*-,,..*;'!%.G!H!!!$"
 	&&sDL999K"CNN$4$4dl	
 	
 	
 !+,,,,,r"   ro   rq   s   @r    r   r     sG        6 6 6 6 6- - - - - - -r"   r   c                   $     e Zd Z fdZd Z xZS )r   c                     t                                                       || _        || _        t	                      | _        d S rM   )rh   rQ   r   r  rs   r1  rE  s      r    rQ   zDenseTable.__init__;  s6     

r"   c                    | j         }|                                s9t          |                                          dk     s|                                rd S |                                |_        t          j        |_        d|_	        d|_
        d|j        _        |                                d         |j        _        d|j        _        d| j        _        | j                            || j                   | j                            | j        j        |                                | j                   | j        d         rdnd	| j        _        | j                            |j                   d S )
Nr:   MemoryDenseTabler0  r8  r   MergedDenser:  TF)r  rO  r   r   r   r=  r   PS_DENSE_TABLEr/   r'   r   r1  rC   r   rB   rE   r2  r   r  r   r   r   r   r_   r\  s      r    r_   zDenseTable._setA  s=   h!!	3&&(())A-- . F"||~~)8"4 #.A+'*||~~a'8$*+'!.&&sDL999K"CNN$4$4dl	
 	
 	
 $(<	#:E44+,,,,,r"   ro   rq   s   @r    r   r   :  sG        # # # # #- - - - - - -r"   r   c                       e Zd Zd Zd ZdS )Serverc                     d S rM   rN   rO   s    r    rQ   zServer.__init___  rR   r"   c                     d S rM   rN   rO   s    r    r_   zServer._setb  rR   r"   Nr`   rN   r"   r    rd  rd  ^  2              r"   rd  c                   $     e Zd Z fdZd Z xZS )DownpourServerc                 H    t                                                       d S rM   rg   ri   s    r    rQ   zDownpourServer.__init__g  rk   r"   c                     d S rM   rN   rO   s    r    r_   zDownpourServer._setj  rR   r"   ro   rq   s   @r    ri  ri  f  G                  r"   ri  c                       e Zd Zd Zd ZdS )Workerc                     d S rM   rN   rO   s    r    rQ   zWorker.__init__o  rR   r"   c                     d S rM   rN   rO   s    r    r_   zWorker._setr  rR   r"   Nr`   rN   r"   r    rn  rn  n  rg  r"   rn  c                   $     e Zd Z fdZd Z xZS )DownpourWorkerc                 H    t                                                       d S rM   rg   ri   s    r    rQ   zDownpourWorker.__init__w  rk   r"   c                     d S rM   rN   rO   s    r    r_   zDownpourWorker._setz  rR   r"   ro   rq   s   @r    rr  rr  v  rl  r"   rr  c                       e Zd Zd Zd ZdS )fsClientc                     || _         d S rM   )fs_client_param)rP   rx  s     r    rQ   zfsClient.__init__  s    .r"   c                     t          j        | j                  sd S | j        j        |_        | j        j        |_        | j        j        |_        | j        j        |_        d S rM   )r   MessageToStringrx  uriuserpasswd
hadoop_binr   s     r    r_   zfsClient._set  sX    *4+?@@ 	F(,	).
+2/:r"   Nr`   rN   r"   r    rv  rv  ~  s2        / / /; ; ; ; ;r"   rv  c                   >    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	S )
PsDescBuilderc                    || _         |d         | _        |d         | _        |d         | _        |d         | _        d | _        t          | j         | j                  | _        i | _        g | _	        | 
                                | _        |                                 | _        |                                 | _        t!          j                    | _        t!          j                    | _        d S )Nr:  r   r9  ry   split_dense_table)r   r:  r   r9  ry   barrier_table_idget_the_one_send_contextrF  tensor_table_dict_server_sub_program_get_tablestables_get_serviceservice_get_fs_client	fs_clientr   PSParameterps_descFLParameterfl_desc)rP   r   s     r    rQ   zPsDescBuilder.__init__  s    y)y) '(: ;!,/ $0LD,A
 
 
 "$#% &&((((**,,..%133%133r"   c           	      .   d}| j         s+| j                            t                      j                   g }| j         D ]V}|                     t                      d         t          |          t          | j        d                              |dz  }W|S )Nr   r   r   r:   )	r  r  r   r   descglobalsr   r%  r   )rP   program_idxr  r   s       r    _get_tensor_tablesz PsDescBuilder._get_tensor_tables  s    % 	<$++GIIN;;;0 	 	JMM(		-(KKdl<.H   
 1KKr"   c                    g }t          | j                                                  D ]X\  }\  }}t          d|||           |                                r| j        t          j        k    r| j        d         r|d d         | j        d         v s| j        d         s7|	                     t                      d         | j        |                     |	                     t                      d         | j        |                     |	                     t                      d         | j        |                     | j        s6|	                     t                      d         | j        |                     Z|                                 | _        |                    | j                   |	                     t                      d         | j        t          |                               |S )Nzidx, name, ctx:local_sparser   r   r   r   )r   rF  itemsrA   r   r   r  r  r   r   r  ry   r  tensor_tablesr   r   )rP   r  r5   r?   r  s        r    r  zPsDescBuilder._get_tables  s    )$-*=*=*?*? @ @ 	N 	NC$#S$444}} N<?#666^4
 "In)EEE"l>: F7GII&67cJJ    4GIIm4T\3GG    MM":'))M":4<"M"MNNNN NMM"9'))L"9$,"L"LMMM!4466d()))/gii/c&kkJJKKKr"   c                 H    | j         rt                      S t                      S rM   )ry   re   rK   rO   s    r    r  zPsDescBuilder._get_service  s     ? 	<<99r"   c                 @    t          | j        d         j                  S )Nr   )rv  r   rx  rO   s    r    r  zPsDescBuilder._get_fs_client  s    %<=MNNNr"   c                     d S rM   rN   )rP   client_infos     r    build_fl_client_descz"PsDescBuilder.build_fl_client_desc  rR   r"   c                 6   | j         D ]}| j        j        j        j                                        }|                    |           | j        j        j        j                                        }|                    |           t          |          t          k    r| j        |j        | _        | j                            | j        j        j        j                   | j                            | j        j                   t#          j        | j                  S rM   )r  r  worker_paramdownpour_worker_paramdownpour_table_paramrR  r_   server_paramdownpour_server_paramr/   r   r  r5   r  service_paramr  rx  r   rz  rP   tabler5  s      r    build_worker_desczPsDescBuilder.build_worker_desc  s    [ 	2 	2E,3I^bbddKJJ{###,3I^bbddKJJ{###E{{l**t/D/L(-	%L%;I	
 	
 	
 	DL8999*4<888r"   c                    i | _         | j        D ]t}| j        j        j        j                                        }|                    |           |j        t          j
        k    r |j        |j        | j         |j        j        <   u| j                            | j        j        j        j                   | j                            | j        j                   t%          j        | j                  S rM   )sparse_table_mapsr  r  r  r  r  rR  r_   r/   r   rP  r2  r=  r   r  r  r  rx  r   rz  r  s      r    build_server_desczPsDescBuilder.build_server_desc  s    !#[ 		 		E,3I^bbddKJJ{### N$BBB&2  ( &{'9'DE 	L%;I	
 	
 	
 	DL8999*4<888r"   N)ra   rb   rc   rQ   r  r  r  r  r  r  r  rN   r"   r    r  r    s        4 4 4.    6  O O O  9 9 99 9 9 9 9r"   r  c                       e Zd Z fdZd Zd Zd Zd Zd Zd%dZ	d%d	Z
d
 Zd&dZd Zd Zeg fd            Zd Z	 d%dZd Z	 d'dZ	 	 	 d(dZd Zd Zd Zd Z	 d)dZd Zd Zd Zd Zd Zd  Z d! Z!d" Z"d# Z#d%d$Z$ xZ%S )*TheOnePSRuntimec                     t                                                       d | _        d | _        t	          j                    | _        d | _        g | _        d | _	        d | _
        d S rM   )rh   rQ   _communicator_serverr   DistFleetWrapper_worker_coordinatorr  _heter_client	_send_ctxri   s    r    rQ   zTheOnePSRuntime.__init__  s[    !,.. #% !r"   c                 j   || _         |d         | _        t          | j                  | _        t	          t          t          j        dd                              | _        |d         | _	        |
                    d| j	        g          | _        | j        | j         d<   |
                    d|d         g          | j         d<   | j        j        | j         d<   | j         d         | _        t          |d	                   | j         d
<   | j         d
         j        | j         d<   |d	         j        d         | j         d<   |d	         j        d         | j         d<   | j         d         t"          j        k    rdnd| j         d<   i | j         d<   i | j         d<   |d         j        d         | j         d<   |d         j        d         | j         d<   t)          d                    | j         d         | j         d                              t-          | j                    t/          | j                  | _        t3          | j                  | _        g | _        t9          | j                  D ]l\  }}|                    d          \  }}t=          j        |t          |          |          }| j                             |!                                           m| j        j"        | _#        g | _$        | j#        rt)          d| j                    | j        %                                }t9          |          D ]l\  }}|                    d          \  }}t=          j        |t          |          |          }| j$                             |!                                           mtM          | j                   | _'        d S )Nr   PSERVER_DEBUG0origin_main_programr   r   origin_startup_programr9  valid_strategytrainerr   ry   rN  TFr:  r   tensor_tabler   r  remote_sparsez+fl-ps > local_sparse: {}, remote_sparse: {}:zfl-ps > all ps addrs: )(r   r   r   role_idboolr   osgetenvdebugr  r,  r   _is_heter_parameter_server_moder9  TrainerRuntimeConfigmodea_sync_configsr  SYNCtrainer_desc_configsrA   formatbuild_var_distributedget_trainer_endpointstrainer_endpointsr  	endpointsstring_hostsr   r   r   PSHostr   serialize_to_string_with_coordinatorwith_coordinatorcoordinator_hosts_get_coordinator_endpointsr  ps_desc_builder)	rP   r   r5   ephostportpshostcoordinator_endpointsips	            r    _set_basic_infozTheOnePSRuntime._set_basic_info  s   !,/"4?33#bi==>>??
#*+@#A $+KK"T%=$>%
 %
! 04/H+,29++%0H(I'J3
 3
./ O; 	'( !%-? @"6$%#
 #
Y #',y"9">Y%,-=%>%M&
\" )0)

)*_% L+/CCCDD 	Y 35./')^$'.#(

~(/^$ )0#)

)0_% 	9@@^,dl?.K 	
 	
 	
 	dl+++!6t!G!G)$/:: 00 	C 	CGC#JD$[s4yy#66F$$V%?%?%A%ABBBB $ A!#  	L>4+<>>???$(O$N$N$P$P!$%:;; L LR88C==DRTC88&--f.H.H.J.JKKKK,T\::r"   c                 h   g }|                                 D ]\  }}|                                rt          | j        |                                          \  }}}||         }	|                                }
||
         }| j                            |	|
|           |                    |           |S rM   )	r  r   r!   r   r   r=  r  push_dense_paramsr   rP   scopesrF  recv_mapall_var_namesr?   r  _r5   scoper=  	var_namess               r    _init_all_paramsz TheOnePSRuntime._init_all_paramsJ      !)) 		, 		,ID#}} )$,8H8HIIIAq#3KE||~~H *IL**5(IFFF  ++++r"   c                 h   g }|                                 D ]\  }}|                                rt          | j        |                                          \  }}}||         }	|                                }
||
         }| j                            |	|
|           |                    |           |S rM   )	r  r   r!   r   r   r=  r  pull_dense_paramsr   r  s               r    _pull_all_densezTheOnePSRuntime._pull_all_denseX  r  r"   c                 N   g }|                                 D ]\  }}|                                r|                                t          |          k    r@|                                }||         }	| j                            |||	           |                    |	           |S rM   )r  r   r   r   r=  r  r  r   
rP   r   r  rF  r  r  r?   r  r=  r  s
             r    _init_paramszTheOnePSRuntime._init_paramsf      !)) 		, 		,ID#}} ~~2g;;..||~~H *IL**5(IFFF  ++++r"   c                 N   g }|                                 D ]\  }}|                                r|                                t          |          k    r@|                                }||         }	| j                            |||	           |                    |	           |S rM   )r  r   r   r   r=  r  r  r   r  s
             r    _pull_densezTheOnePSRuntime._pull_denset  r  r"   Nc                 2     j                                         }g }t           j        d         t                    ryt           j        d                   dk    r[t          t           j        d                             D ]2}|                     j        d         |         j        j	                   3n*|                     j        d         j        j	                   t          t          |                    D ]} j        d         rm||         j
        si ||         _
        d||         j
        d<   t          j        d          }d |                    d          D             }|||         j
        d<    j        d	         rd||         j
        d	<    fd
}t           j         j                  }t!           j         j         j                  }| _         j        d         }	 j        r_t)          d|            t)          d           |D ]}
t)          |
 d||
                     |D ]}
t)          |
 d||
                     i }d|d<    j                                        |d<    j                                        |d<    j         j        |d<    j        d         t2          j        k    r |            }|                    |           t)          d|	                                            j                            | j         j                     j        s0tC           j                   _"        t)          d j"                    t)          d j#                    t)          d j$                     j#        r+ j        %                     j$         j          j"                    j        d         t2          j&        k    s j        rltO          |	j(        ||	                                           _)         j)        *                    ||| j        tV          j,        -                                           t\          j/        0                                  j        1                                }t          |t                    rt          |          dk    r j        2                    |d                   }t          |t                    stg          j4        d           |g} j        5                    |            j        6                                 t)          d           nt)          d            j        d         }to          tq          t          j        d d                              }|Ft           j9                  dk    rtu          d!          tV          j,        -                                g}t           j9                  t          |          k    rtu          d"          | _;        |s j        d         t2          j&        k    s j        r j)        <                    |           n> j        d         s1 j         dk    r&t)          d#            =                    |||           t\          j/        0                                  j        d         s >                    |||           t\          j/        0                                  j        d         t2          j&        k    s j        rG j)        ?                                s j)        @                                 ntg          j4        d$           |jA        d%         }tq          t          j        d&d'                    }|r|rt           j        C                                            j        rC j        D                                g k    r&t           j        D                                            j        rg } j        E                                g k    r j        E                                }g } j        D                                g k    r j        D                                }t          || j                                                   _G        d S d S d S d S )(Nlossr:   ry   TFLAGS_selected_gpusc                 ,    g | ]}t          |          S rN   r   )r   ss     r    r   z0TheOnePSRuntime._init_worker.<locals>.<listcomp>  s    @@@qCFF@@@r"   ,worker_placesrN  c                  |    i } j                                         | d<   j                                         | d<   | S )Npserver_endpoints
trainer_id)r   _get_pserver_endpoints_worker_index)kwargsrP   s    r    sync_strategy_envsz8TheOnePSRuntime._init_worker.<locals>.sync_strategy_envs  sC    F6688 &' $(?#@#@#B#BF< Mr"   r  r  ep_listr  zworker_desc: 
zcommunicator send_ctx:z: r  need_global_stepr  trainersr  r   zcommunicator config:zfl-ps > trainer_endpoint: zfl-ps > with_coordinator? zfl-ps > coordinator addr: r   z!gloo may not initialize correctlyzcreate c2c connection donezcannot create c2c connectionr  	TEST_MODEz;You must set the scope list when you have Multiple programszlen(programs) != len(scopes)z entering self._init_all_params()z'communicator has been initialized, skiplaunch_barrierFLAGS_LAUNCH_BARRIER1)Hr  r  
isinstancer   listr   ranger   blockr   
_fleet_optr  r  r   get_the_one_recv_contextr9  r  r  r  r  rA   r   _role_id_worker_numr  r  r  updateget_communicator_flagsr  init_workerr  r  get_trainer_endpointtrainer_endpointr  r  init_fl_workerr  r   r  r  init_with_ctxpaddlestaticglobal_scoper   utilbarrierget_client_info_all_gatherr  r  set_clientscreate_client2client_connectionr  r   r   rD   r  init_paramsr  r  
is_runningstartr  r   r  _get_next_trainers_get_previous_trainersr   r  )rP   r  worker_descmain_programsr   gpus_envr  	dense_maprF  trainer_configkeyr  sync_kwargsinfoall_infodist_strategyis_testr  launch_barrier_flagprevious_trainersnext_trainerss   `                    r    _init_workerzTheOnePSRuntime._init_worker  s   *<<>>t|F+T22	EDL())A--3t|F34455 L L$$T\&%9!%<%B%JKKKKL   f!5!;!CDDDs=))** 		D 		DA|L) H$Q'2 524M!$/<@a +L99%:;;@@HNN3,?,?@@@?Ga +O<|O, D?Ca +O<	 	 	 	 	 -LD,A
 
 
	 ,L"3N
 
 

 "i0: 	21K11222*+++ 1 1////0000  2 200	#001111%(!"#7799|!_88::z%)%9%J!"<	"o&:::,,..KMM+&&&$n&K&K&M&MNNN  d.?NNN$ 	H$8$I$ID!Ft/DFFGGGB4+@BBCCCC4+ACCDDD  	L''&d6K  
 L#':::$ ; ".#5577" "D
 ,,!**,,   	
 |++--dD!! 	2c$ii!mm22Q H h-- &ABBB$: L$$X...L88:::.////0111%56s29[#667788>4,--11 Q   m00223Ft())S[[88;<<< 	!Y'?+>>>( ? "..y9999|L1 K|q((@AAA--fh	JJJJ   |L) 	>  9===
 L#':::$ ; %0022 I"((****GHHH&56FG!"),BC"H"HII 	1 	doDDFFGGG%HO6688B>>!$/"D"D"F"FGGG$ 	$&!?99;;rAA(,(N(N(P(P% "?55772==$(O$F$F$H$HM%0!#4do6N6N6P6P& &"""	 	 	 		 	r"   c                    | j         t          | j                  | _         t          d| j        d                     t          d| j                    | j                             | j        d         | j                   d S )Nz>>> curr node ip: r   z>>> all trainer endpoints: )r  r
   r  rA   r  r  start_coordinator)rP   r  s     r    _init_coordinatorz!TheOnePSRuntime._init_coordinator.  s    $ +D,= > >D>4#9!#<>>???DD,BDDEEE++"1%t'=	
 	
 	
 	
 	
r"   c                 J    | j         d S | j                                          d S rM   )r  make_fl_strategyrO   s    r    _make_fl_strategyz!TheOnePSRuntime._make_fl_strategy8  s,    $33..00000r"   c                    | j                                         }t          | j                  }| j        r)|t          | j                                                  z  }| j        rt          d|            t          j
                    | _        | j                            || j        | j        || j                   t!          | j        d          }t!          | j        d          }||z   }||}	n|D ]}
|
|vrt%          d|           |}	||	sd S | j         j        }t(          j                            |          }| j                                        }|	D ]&}
||
         }| j                            |d|           'd S )Nzserver_desc: 
TFz4fleet.init server can only load sparse variables in r  )r  r  r  r   r9  r   r?  r  rA   r   r  r  init_serverr  r  r  get_sparse_tablenamesr   rD   r  r  pathnormpathr  load_sparse)rP   dirnamer  r  server_descr  dist_varnamessparse_varnamesdistributed_varnamesload_varnamesvar_namer  r   r=  s                 r    _init_serverzTheOnePSRuntime._init_server>  s   *<<>>00  	KDOGGIIJJJH: 	31K11222,..  L$	
 	
 	
 .d.GNN/%u
 
  ->0MM%  #777$eOcee   8 &M?-?F 0B'""7++_--//
% 	= 	=H(2HL$$Wc8<<<<	= 	=r"   c                     t          | j                  }|                    d          \  }}| j                            |t          |                     d S )Nr  )get_ps_endpointr   r   r  
run_serverr   )rP   r  r  r  s       r    _run_serverzTheOnePSRuntime._run_serverm  sH    T_--XXc]]
dc$ii00000r"   c                    | j         d         t          j        k    r| j                                         | j                                         | j        r,| j        
J d            | j                                         d S d S )Nr   z/heter client should not be None in heterps mode)	r   r  r  r  stopr  stop_workerr9  r  rO   s    r    _stop_workerzTheOnePSRuntime._stop_workerr  s    <	"o&999##%%%  """  	&%11A 211 ##%%%%%		& 	&r"   c                       fd}|S )Nc                    | j         v rdS ddlm}  || j                   \  }}}|                    d          rdS |                    d          rdS | j                                        t          j        j	        j
        k    sb| j                                        t          j        j	        j        k    s1| j                                        t          j        j	        j        k    rdS | j        S )NFr:   )_get_varname_partsz@GRADr   )r?   utils.publicrU  endswith
startswithr  r/   r   VarDescVarTypeFEED_MINIBATCH
FETCH_LISTREADERpersistable)rH   rU  origin_varnamer  exclude_var_namess       r    is_validz0TheOnePSRuntime.__exclude_vars.<locals>.is_valid~  s    x,,,u888888#5#5ch#?#? NAq&&w// u(()9:: u 4<#7#FFF8==??dl&:&EEE8==??dl&:&AAAu?"r"   rN   )r`  ra  s   ` r    __exclude_varszTheOnePSRuntime.__exclude_vars|  s#    	# 	# 	# 	# 	#* r"   c                     |                     d          s|                     d          rd}n t          j                            |d          }|S )Nzafs:zhdfs:z./dnn_plugin
dnn_plugin)rX  r  r@  r   )rP   rC  
model_paths      r    _get_inference_model_pathz)TheOnePSRuntime._get_inference_model_path  sO    f%% 	=););G)D)D 	='JJg|<<Jr"   c                    t          | j        | j                  }t          | j        | j        | j                  }t          | j                  dk    r| j        d         |                     |||          }||n|}	fd|	D             }
dd l}|j	        
                    |          5  |j	                            |d|
|           d d d            d S # 1 swxY w Y   d S )Nr  r  r:   r   c                 ^    g | ])}                                                     |          *S rN   r*   rH   )r   r   r   s     r    r   z9TheOnePSRuntime._ps_save_dense_params.<locals>.<listcomp>  s3    FFF!$$&&**1--FFFr"   z./)r
  filename)r  r   r9  r  r  r   r   r  r  r  scope_guard	save_vars)rP   executorrC  r  r   r  r+  rF  dense_var_namessave_var_namesr
  r  s       `       r    _ps_save_dense_paramsz%TheOnePSRuntime._ps_save_dense_params  s_    -LD,A
 
 
	 ,L"3N
 
 

 ?c$";<<AA/2G**7E8YOO,5,=9FFFF~FFF]&&u-- 	 	M##$dW $   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   / CC #C c                 P   t          | j        d          }g }|                     |          }|                                D ]d\  }	}
|
d         |vr$	 | j                            |	|           n#  Y nxY w| j                            |	||           |                    |
           e|S NTr   )r?  r   rf  r  r  recv_and_save_modelsave_one_modelr   )rP   rm  rC  r   r3   r  rG  valuesre  r   namess              r    _save_sparse_paramsz#TheOnePSRuntime._save_sparse_params  s      5%t 
  
 33G<<
  		! 		!IBQx333L44RDDDDDL''GT:::MM%    s   A--A1r   c                     t          |t                    st          d          || j        d         }t          |t                    rt          d          | j                            ||           dS )a  
        This function filters out all variables with `persistable==True` from the
        give `main_program` and then saves these variables to the folder `dirname`
        or file `filename`.

        The `dirname` is used to specify the folder where persistable variables
        are going to be saved. If you would like to save variables in separate
        files, set `filename` None; if you would like to save all variables in a
        single file, use `filename` to specify the file name.
        ;in fleet.save() function, executor must be as Executor typeNr  ^in fleet.save() function, main_program must be as Program type, CompiledProgram is not allowed)r  r   	TypeErrorr   r   r  save_all_model)rP   rm  rC  r3   r  r  s         r    _save_distributed_persistablesz.TheOnePSRuntime._save_distributed_persistables  s     (H-- 	M   <(=>LlO44 	p   	##GT22222r"   Tc                    t          |t                    st          d          ddl}|| j        d         n|t          | j        t                              \  }	}	}
| j        |
         }t          d|
           t          t                    rt          d          fd|D             }|j                            ||          }|                               |                     |          }d}t          j                            ||          } |j        ||           t'          | j        d| j        	          }|                     |||||          }t'          | j        | j        
          }t-          | j        | j        | j                  }|                     |||           | j        d         j        d         }t5          |          }t5          t7          t8                              |          |                                                    }|D ]M}|                    |          } |j        |t          j                            ||j                   d           NdS )z
        Prune the given `main_program` to build a new program especially for inference,
        and then save it and all related parameters to given `dirname` by the `executor`.
        ry  r   Nzsave inference model scope idx:rz  c                 ^    g | ])}                                                     |          *S rN   ri  )r   r?   r   s     r    r   zFTheOnePSRuntime._ps_inference_save_inference_model.<locals>.<listcomp>	  s@     
 
 
15G  ""&&t,,
 
 
r"   	__model__Fis_denser  r  r  r   stat_var_namesT)use_binary_format)!r  r   r{  r  r   r!   r   r   r  rA   r   r  normalize_program_copy_dist_param_info_fromrf  r  r@  r   saver  r9  rw  r  r  r  r  r  filterr  _TheOnePSRuntime__exclude_varsr>   	get_valuer?   )rP   rm  rC  feeded_var_namestarget_varsr3   export_for_deploymentr  r  r  r5   r  	feed_varsinfer_programre  model_basenamesparsessparse_namesr+  rF  generate_varsremaining_varsrH   r3  r   s                           @r    "_ps_inference_save_inference_modelz2TheOnePSRuntime._ps_inference_save_inference_model  s    (H-- 	M   	 # %a(( 	
 &dlBwKK@@	1cC /555g// 	p  
 
 
 
9I
 
 
	 77Y
 
 	0099933G<<
$j.AAM>222*L"3
 
 

 //gwd
 
 -LD,A
 
 
	 ,L"3N
 
 

 	%9===#

/1 ]++..|<<'')) 
 
 " 	 	C]]5))FFKZ22"&    	 	r"   c                    |                     dd          }|                     dd          }| j                                         t          j                                         d}| j                                        r| j                            |          }t          j                                         | j                                        r| j        	                    ||||           t          j                                         d}| j                                        r| j        
                    |||          }t          j                                         |S )Nr  r:   r=  r   g        r   )r,  r  client_flushr   r  r  r   _is_first_workerget_cache_thresholdcache_shuffle
save_cache)rP   rC  r  r  r=  cache_thresholdfeasign_nums          r    _save_cache_modelz!TheOnePSRuntime._save_cache_model>  s+   zz&!$$::j!,,!!###
?++-- 	I"l>>xHHO
?++-- 	QL&&x$PPP
?++-- 	K,11(GTJJK
r"   c                    t           j                                         | j        d         s| j                                        r| j                            |||           t           j                                         d S Nry   )r   r  r  r   r   r  r  save_cache_table)rP   r=  pass_idmem_cache_key_thresholds       r    _save_cache_tablez!TheOnePSRuntime._save_cache_tableV  sx    
<% 	)I)I)K)K 	L))'#:   	
r"   c                     t           j                                         | j                                        r| j                                         d S d S rM   )r   r  r  r   r  r  check_save_pre_patch_donerO   s    r    _check_save_pre_patch_donez*TheOnePSRuntime._check_save_pre_patch_done^  sP    
?++-- 	5L2244444	5 	5r"   c                    t          | j        d          }g }|                                D ]T\  }}|d         |vrt          j        d           | j                            |||           |                    |           U|S )NTr   z,varname is not in distributed_varnames, pass)r?  r   r  r  r  r  load_one_tabler   )	rP   rC  r   r3   r  rG  ru  r   rv  s	            r    _load_sparse_paramsz#TheOnePSRuntime._load_sparse_paramsc  s    4%t 
  
   	! 	!IBQx333LMMML''GT:::MM%    r"   c                    || j         d         n|}t          | j        t          |                    \  }}}| j        |         }t          d|           t          |t                    rt          d          t          | j        d| j
                  }|                     ||||          }t          | j        | j
                  }	t          | j        | j
        | j                  }
g }|	                                D ]\  }}|                    |           |}t!          t#          t$                              |          |                                                    }|                     |          }dd l}|D ]U}|j        |vr |j        t2          j                            ||j                            }|                    ||           V|                     |||
|	           d S )Nr   zload inference model scope idx:rz  Fr  r  r  )r   r!   r   r   r  rA   r  r   r{  r  r9  r  r  r  r  r   r  r  r  r  r>   rf  r  r?   loadr  r@  r   	set_valuer  )rP   rC  r  r3   r  r5   r  r  rF  r+  rF  recv_dense_varnamesrv  loaded_varnamesr  re  r  rH   r3  s                      r    "_ps_inference_load_inference_modelz2TheOnePSRuntime._ps_inference_load_inference_modelq  s&   
 # %a(( 	
 &dlB|4D4DEE	1cC /555lO44 	p   +L"3
 
 
 22WlD
 
 -LD,A
 
 
	 ,L"3N
 
 
 !!)) 	. 	.HAu&&u----)..??&&(( 
 
 33G<<
! 	) 	)Cx222 V[j#(!C!CDDFMM&%((((,xCCCCCr"   c                     t           j                                         | j                                        r| j                            |||           t           j                                         d S rM   )r   r  r  r   r  r  rt  rP   r=  r@  r  s       r    _save_one_tablezTheOnePSRuntime._save_one_table  `    
?++-- 	>L''$===
r"   c                     t           j                                         | j                                        r | j        |i | t           j                                         d S rM   )r   r  r  r   r  rp  rP   argsr  s      r    _save_dense_paramsz"TheOnePSRuntime._save_dense_params  s^    
?++-- 	8&D&7777
r"   c                     t           j                                         | j        d         s| j                                        r | j        |i | t           j                                         d S r  )r   r  r  r   r   r  r}  r  s      r    _save_persistablesz"TheOnePSRuntime._save_persistables  sn    
<% 	A)I)I)K)K 	A/D/@@@@
r"   c                     t           j                                         | j                                        r | j        |i | t           j                                         d S rM   )r   r  r  r   r  r  r  s      r    _save_inference_modelz%TheOnePSRuntime._save_inference_model  s_    
?++-- 	E3D3TDVDDD
r"   c                     t           j                                         | j                                        r| j                            |||           t           j                                         d S rM   )r   r  r  r   r  r  r  r  s       r    _load_one_tablezTheOnePSRuntime._load_one_table  r  r"   c                     t           j                                         | j        d         s| j                                        r| j                            ||           t           j                                         d S r  )r   r  r  r   r   r  r  
load_modelrP   r@  r  s      r    _load_persistablesz"TheOnePSRuntime._load_persistables  sl    
<% 	0)I)I)K)K 	0L##D$///
r"   c                     t           j                                         | j                                        r|                     ||           t           j                                         d S rM   )r   r  r  r   r  r  r  s      r    _load_inference_modelz%TheOnePSRuntime._load_inference_model  s]    
?++-- 	@33D$???
r"   c                     t           j                                         | j                                        r| j                            ||           t           j                                         d S rM   )r   r  r  r   r  r  set_date)rP   r=  day_ids      r    	_set_datezTheOnePSRuntime._set_date  s^    
?++-- 	4L!!(F333
r"   c                     t           j                                         | j                                        r| j                            |||           t           j                                         d S rM   )r   r  r  r   r  r  print_table_stat)rP   r=  r  	thresholds       r    _print_table_statz!TheOnePSRuntime._print_table_stat  sa    
?++-- 	HL))(GYGGG
r"   c                    |t          j        d           nd}t          j                                         | j        d         s| j                                        rVt          | j        d| j        j	                  }|
                                D ] \  }}| j                            ||           !t          j                                         d S )NznThe param threshold is not used in MemorySparseTable, if you need to shrink, please set the config of accessorr   ry   Fr  )r  r  r   r  r  r   r   r  r  r  r  r  shrink_sparse_table)rP   r  r  r   rv  s        r    _shrinkzTheOnePSRuntime._shrink  s     M A    I
<% 	@)I)I)K)K 	@."&/"Q  G %]]__ @ @	E00Y????
r"   rM   )NN)Nr   rr  )r   N)&ra   rb   rc   rQ   r  r  r  r  r  r6  r9  r<  rJ  rN  rR  staticmethodr  rf  rp  rw  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rp   rq   s   @r    r  r    s           C; C; C;J        j j j jX
 
 
 
1 1 1-= -= -= -=^1 1 1
& & & )+    \0   <@   .  , :;3 3 3 3F "W W W Wr  0  5 5 5
   -1:D :D :D :Dx                         r"   r  )/r  r  google.protobufr   r  paddle.distributedr   paddle.distributed.communicatorr   r   5paddle.distributed.fleet.base.private_helper_functionr   paddle.distributed.fleet.protor   -paddle.distributed.fleet.runtime.runtime_baser	   !paddle.distributed.ps.coordinatorr
   "paddle.distributed.ps.utils.publicpaddle.frameworkr   paddle.staticr   r   r   __all__r!   r8   rI   rK   re   rs   r   r#  r   r   r   r   r   r   rd  ri  rn  rr  rv  r  r  rN   r"   r    <module>r     sV   
			  ' ' ' ' ' '  $ $ $ $ $ $ E E E E E E E E      : 9 9 9 9 9 E E E E E E 9 9 9 9 9 9 0 0 0 0 ! ! ! ! ! ! < < < < < < < < < <    + + +&  D	- 	- 	- 	- 	- 	- 	- 	-5 5 5 5 5 5 5 5RG RG RG RG RG RG RG RGjy* y* y* y* y*X y* y* y*x	
 
 
 
 
 
 
 
$        5  5  5  5  55  5  5  5F( ( ( ( (% ( ( (.l
 l
 l
 l
 l
% l
 l
 l
^ -  -  -  -  -[  -  -  -F!- !- !- !- !- !- !- !-H           V              V   
; 
; 
; 
; 
; 
; 
; 
;l9 l9 l9 l9 l9 l9 l9 l9^{ { { { {k { { { { {r"   