
    Αi                        S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SKJ	r	  S SK
Jr  SrSrSrS	r\R                   R#                  5       r\R                   R'                  5       r\R                   R*                  R,                  r\R                   R*                  r\R                   R'                  5       r\R                   R*                  R4                  r\R                   R*                  R8                  r\R                   R*                  R<                  r\R                   RA                  5       r!/ S
Qr"/ SQr#SS/r$SSS.r%SSS.r&Sr'/ SQr(\( V s/ s H  o S-   PM	     sn r)S r*Sr+\*" SSS9r, " S S5      r- " S S5      r.S r// 4S jr0S r1S  r2S! r3S" r4S# r5S$ r6S% r7S& r8S' r9S( r:S) r;S* r<S+ r=S, r> S]S- jr?S. r@S/ rAS^S0 jrBS_S1 jrCS2 rDS3 rES4 rFS5 rGS6 rHS7 rI S`S8 jrJS9 rKS: rLS; rMS< rNS= rOS> rPS? rQ SaS@ jrRSbSA jrSSB rT\R                  R                  R                  SC\R                  R                  R                  SD\R                  R                  R                  SE\R                  R                  R                  SC\R                  R                  R                  SD\R                  R                  R                  SE\R                  R                  R                  SF\R                  R                  R                  SF0r_SG r` " SH SI5      raSJ rbSK rcSL rdSM reSN rfSO rgSP rhSQ riSR rjSS rkST rlSU rmSV rnSW roSX rpSY rqSZ rrS[ rsS\ rtgs  sn f )c    N)reduce)generate_control_dev_var_name)is_persistable)coreop_namescopegradient_clipz@PS_STEP_COUNTER@z@LR_DECAY_COUNTER@)cpugpuxpu)sendrecvfetch_barriersend_barrierlookup_tablelookup_table_v2W)r   r   )lookup_table_gradlookup_table_v2_gradr	   )z.batch_sizez
.batch_sumz.batch_square_sum@GRADc                    [         R                  " U5      nUR                  [         R                  S9  [         R                  " U SSSS9nUR                  [         R
                  5        [         R                  " S5      nUR                  U5        [         R                  " 5       nUR                  [         R                  5        UR                  U5        UR                  U5        U$ )N)levelazUTF-8T)modeencodingdelayzD%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s)logging	getLoggersetLevelWARNINGFileHandlerINFO	FormattersetFormatterStreamHandlerDEBUG
addHandler)log_pathlogging_nameloggerhandler	formatterconsoles         b/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/ps/utils/public.pylogger_configr.   :   s    |,F
OO'//O*!!sWDG W\\"!!NI ###%GW]]#
g
gM    z	./ps_log/z./ps_usr_print_logps_usr_print_log)r'   r(   c                   ,    \ rS rSrSrSrSrSrSrSr	Sr
g	)
DistributedModeR   r                   N)__name__
__module____qualname____firstlineno__SYNCASYNC
HALF_ASYNCGEOFLNU__static_attributes__r9   r/   r-   r2   r2   R   s     DEJ
C	
B	
Br/   r2   c                        \ rS rSrS rS rSrg)TrainerRuntimeConfig[   c                 t   S U l         [        R                  " SS5      nUnUR                  S   nUR                  (       d  US:X  a  [
        R                  U l         UR                  (       a  US:X  a  [
        R                  U l         UR                  (       a  US:  a  [
        R                  U l         Un0 U l	        [        R                  " SU5      U R                  S'   [        R                  " SU5      U R                  S'   [        R                  " S	S5      U R                  S
'   [        R                  " SU5      U R                  S'   [        R                  " SS5      U R                  S'   [        R                  " SS5      U R                  S'   [        R                  " SS5      U R                  S'   g )NCPU_NUM1k_stepsr   $FLAGS_communicator_max_merge_var_numcommunicator_max_merge_var_num"FLAGS_communicator_send_queue_sizecommunicator_send_queue_size*FLAGS_communicator_independent_recv_thread$communicator_independent_recv_thread0FLAGS_communicator_min_send_grad_num_before_recv*communicator_min_send_grad_num_before_recv#FLAGS_communicator_thread_pool_size5communicator_thread_pool_size"FLAGS_communicator_send_wait_timescommunicator_send_wait_times#FLAGS_communicator_is_sgd_optimizercommunicator_is_sgd_optimizer)
r   osgetenva_sync_configsa_syncr2   r>   r?   rA   runtime_configs)selfvalid_strategynum_threadssend_queue_sizerK   s        r-   __init__TrainerRuntimeConfig.__init__\   s   	ii	3/% //	:$$A',,DI  W\'--DI  Wq['++DI%O!AC2OB
=> @Byy0/@
;< IIBCH 	CD IIBK 	IJ
 AC		13A
<= @Byy0#@
;< AC		13A
<=r/   c                 n   / n[         R                  " SS5      nSnU R                  b  U R                  [        R                  :X  a  U R
                  R                  5       nSnOsU R                  [        R                  :X  d  U R                  [        R                  :X  a  Sn/ SQnO0U R                  [        R                  :X  a  Sn/ SQnO[        S	5      eU R                  [        R                  :X  d  U R                  [        R                  :X  av  U R
                  S
   nU R
                  S   nXB:w  a'  [        SU SU SU SU S3	5        UU R
                  S
'   XR:w  a'  [        SU SU SU SU S3	5        UU R
                  S'   U Vs0 s H  of[        U R
                  U   5      _M     sn$ s  snf )NrI   rJ    asynczsync or half_async)rM   rX   rV   rO   rA   )rV   rX   rM   rO   zUnsupported ModerM   rO   zWARNING: In zo mode, communicator_max_merge_var_num must be equal to CPU_NUM. But received, communicator_max_merge_var_num = z, CPU_NUM = z3. communicator_max_merge_var_num will be forced to .zk mode, communicator_send_queue_size must be equal to CPU_NUM. But received, communicator_send_queue_size = z1. communicator_send_queue_size will be forced to )r[   r\   r   r2   r?   r_   keysr>   r@   rA   
ValueErrorprintstr)r`   	need_keysrb   mode_strmax_merge_var_numrc   keys          r-   get_communicator_flags+TrainerRuntimeConfig.get_communicator_flags   s   	ii	3/99		_-B-B B,,113IHII---yyO666+HI YY/---HI /00 II---yyO666 $ 4 40! #22.O !/"8* -88I7J,"m#VWbVccdf   $$%EF -"8* -66E5Fl"m#TU`Taabd   $$%CD @IIyS--c233yIIIs   "F2)r   r_   N)r:   r;   r<   r=   rd   rr   rD   r9   r/   r-   rF   rF   [   s    '
R<Jr/   rF   c                 0   / n[        U R                  5       R                  5       Hl  u  p#[        UR	                  [
        5      5      nU[        [        5      :X  d%  U[        [        5      [        [        5      -  :X  d  M[  UR                  U5        Mn     U$ N)		enumerateglobal_blockopsintattrRPC_OP_ROLE_ATTR_NAMELR_SCHED_OP_ROLE_ATTR_VALUEOPT_OP_ROLE_ATTR_VALUEappend)programlr_opsindexoprole_ids        r-   
get_lr_opsr      s    Fw33599:	bgg345c566'S'F
&'F( ;( MM" ; Mr/   c                    U R                  5       n/ nUR                   H  n[        U5      (       d  M  [        U5      S:  a  UR	                  S5      S   U;  a  M>  [
        UR                  5       ;   a\  [        UR                  [
        5      ;   a?  UR                  S[        [        R                  R                  R                  5      5        M  UR                  U5        M     U$ )Nr   Paramop_role)rw   rx   _is_opt_role_opleninputOP_NAME_SCOPE	all_attrsCLIP_OP_NAME_SCOPErz   	_set_attrry   r   op_proto_and_checker_makerOpRoleBackwardr~   )_programremote_sparseblockopt_opsr   s        r-   get_optimize_opsr      s    !!#EGii2M"Q&HHW%a(= /&"''-*@@77>>GGH NN2# $ Nr/   c                     U R                  5       n/ nUR                   H&  nUR                  S:X  d  M  UR                  U5        M(     U$ )N	data_normrw   rx   typer~   )r   r   r   r   s       r-   get_datanorm_opsr      sB    !!#EGii77k!NN2  Nr/   c                     [        [        R                  " SS5      5      n SnSnSn[        R                  " S5      (       aD  [        R                  " S5      nUR                  S5      U    n[	        UR                  S5      5      nU UUUS.$ )NPADDLE_TRAINER_ID0rg   r   PADDLE_TRAINER_ENDPOINTS,)
trainer_idnum_trainerscurrent_endpointtrainer_endpoints)ry   r[   r\   splitr   )r   r   r   r   s       r-   get_dist_envr      s    RYY2C89JL	yy+,,II&@A,2237
C,22378 !$,.	 r/   c                 d     U R                  5       $ ! [         a    U R                  5       s $ f = fru   )_role_id	Exceptionr   
role_makers    r-   get_role_idr     s4    $""$$ $!!##$    //c                      U R                  5       [        U 5         $ ! [         a    U R                  5       [        U 5         s $ f = fru   )_get_pserver_endpointsr   r   get_pserver_endpointsr   s    r-   get_ps_endpointr   	  sK    K002;z3JKK K//1+j2IJJKs    &AAc                 d     U R                  5       $ ! [         a    U R                  5       s $ f = fru   )r   r   r   r   s    r-   get_ps_endpointsr     4    20022 2//112r   c                 "    U R                  5       $ ru   )_get_heter_worker_endpointr   s    r-   get_heter_worker_endpointr     s    0022r/   c                 "    U R                  5       $ ru   )_get_trainer_endpointr   s    r-   get_trainer_endpointr     s    ++--r/   c                 "    U R                  5       $ ru   )_get_trainer_endpointsr   s    r-   get_trainer_endpointsr     s    ,,..r/   c                 d     U R                  5       $ ! [         a    U R                  5       s $ f = fru   )_get_previous_trainersr   get_previous_trainersr   s    r-   get_previous_stage_trainersr   #  r   r   c                     U R                   [        ;   a  U R                  S5      SL a  gU R                   S:X  a  U R                  S5      SL a  gg)Nis_distributedTdistributed_lookup_tableFr   SPARSE_OP_LISTrz   r   s    r-   is_distributed_sparse_opr   *  sI    	ww. RWW-=%>$%F 	--GG$%-r/   c                 *    U R                  S5      S   $ )Nr   r   )r   r   s    r-   get_sparse_tablenamer   7  s    88C=r/   c                     U R                   [        ;   a)  U R                  S5      SL a  U R                  S5      SL a  gU R                   S:X  a  U R                  S5      SL a  gg)N	is_sparseTr   Fr   r   r   s    r-   is_sparse_opr   ;  s[    
>!GGK D(GG$%. 	--GG$%.r/   c                    [        5       nU  H  nU(       aO  UR                  5       R                   H/  n[        U5      (       d  M  UR	                  [        U5      5        M1     MY  UR                  5       R                   H/  n[        U5      (       d  M  UR	                  [        U5      5        M1     M     [        U5      $ ru   )setrw   rx   r   addr   r   list)programsr   
tablenamesr   r   s        r-   get_sparse_tablenamesr   L  s    J**,00+B//NN#7#;< 1 **,00##NN#7#;< 1  
r/   c                 d     U R                  5       $ ! [         a    U R                  5       s $ f = fru   )_worker_numr   
worker_numr   s    r-   get_trainersr   Z  s4    '%%'' '$$&&'r   c                    [        U5      S:  a  U$ U(       Gd  / n/ nU Hr  nSn	US   n
U
R                  R                  n[         H  nUR	                  U5      (       d  M  Sn	M     U	(       a  UR                  U5        Ma  UR                  U5        Mt     / nSnU Hy  nUS   n
UR                  U
R                  R                  5        U R                  5       R                  U
R                  R                     nU[        S UR                  S5      -  nM{     S[        U5      -   nSnSSKJn  U" UU/S/U/UUUSSUSS[        U 5      / 5      nUUU'   US-  n[        U5      S::  a  U$ / nSnU Hy  nUS   n
UR                  U
R                  R                  5        U R                  5       R                  U
R                  R                     nU[        S	 UR                  S5      -  nM{     S
[        U5      -   nSnSSKJn  U" UU/S/U/UUUSSUSS[        U 5      / 5      nUUU'   US-  nU$ U H  nUS   n
U
R                  R                  nU R                  5       R                  U   n[        S UR                  S5      nUnSnSSKJn  U" UU/S/U/U/UUSSUSS[        U 5      / 5      nUUU'   US-  nM     U$ )Nr4   FTr   c                 
    X-  $ ru   r9   xys     r-   <lambda>(get_dense_send_context.<locals>.<lambda>      QUr/   zDense@GRAD_CommContext127.0.0.1:6071c                 
    X-  $ ru   r9   r   s     r-   r   r     r   r/   zDataNorm@GRAD_c                 
    X-  $ ru   r9   r   s     r-   r   r         AEr/   )r   
merged_varnameDATA_NORM_GRAD_NAMEendswithr~   rw   varsr   shaperm   paddle.base.corer   id)r   send_ctxidxmerged_dense_pairsr   split_dense_tabledense_pairsdata_norm_pairsmergedis_data_normgradvarnamer   origin_varnames	var_numelvar	grad_name	aggregater   	dense_ctxdata_norm_ctxorigin_varnames                         r-   get_dense_send_contextr  a  s     "
(F L!9Doo**G+##D))#'L , &&v.""6* ) 	!F!9D""4??#7#78&&(--doo.B.BCC 2CIIqAAI	 "
 "CH,		 	1KKwK
	  (q1$J 	%F!9D""4??#7#78&&(--doo.B.BCC 2CIIqAAI	 &
 %s3x/		 	1#KKwK
  ,q: J7 )F!9D!__11N&&(--n=C1399a@I&II4#!" 7I  #,HY1HC5 )6 Jr/   c                 8   U S   [         R                  :w  a  [        SU S    S35      e0 n[        U S   5      nU S   nSn[	        US5      n[        U5       H  u  pgU S   U   nU H  n	U	u  pUR                  R                  nU
R                  R                  nXS	   ;   a  M=  X;   a  SOS
nUR                  5       R                  UR                  R                     n[        S UR                  SS  S5      nSSKJn  [        SUU5        U" UU/S/U/U/USSUUS
S
[        U5      / 5      nUS-  nUUUR!                  5       '   M     M     [#        U5      S:X  a  [        S5      e[#        U S   5      S:  a   U S   (       a  [%        X@S   5      u  nnUUU'   U$ )Nps_modez	ps mode: z) not matched get_geo_trainer_send_contextr   origin_main_programsr   Tmerged_sparse_pairsr   Fc                 
    X-  $ ru   r9   r   s     r-   r   .get_geo_trainer_send_context.<locals>.<lambda>  r   r/   r4   r   z(public get_the_geo_send_context sparse: r   z-GeoSGD require sparse parameters in your net.tensor_table	is_worker)r2   rA   rk   r   r   rv   r   r   rw   r   r   r   r   r   rl   r   var_namer   	_step_ctx)attrsr   r   origin_programsr   distributed_varnamesir   r  r   paramr   r   
param_namer   r   r   r   
sparse_ctxr   ctxs                        r-   get_geo_trainer_send_contextr    s   Y?...i())RS
 	
 HU<01J23O
C0$G0
#$9:1=)F KE,,I))..J?33 #:  &&(--doo.B.BCC1399QR=!DI4:Iy %!"7J  1HC.8HZ((*+E * 1L 8}HII
5 !A%%*<c#67	cOr/   c                     [         n[        U5      n[        U5      nS/[        U5      -  nU/[        U5      -  nSSKJn  U" UUUUU/USSSU SSS/ 5      nX(4$ )Nr4   r   r   TF)STEP_COUNTERr   r   r   r   r   )	r   r   r   r   	endpointssectionsnamesr   r  s	            r-   r  r    s~    DZ(J ,IsS^#HFS^#E,
	

C  9r/   c                    Uc  S/n0 n[        U S   5      nU S   n[        SU 35        Sn[        US5      n[        U5       GHI  u  pU S   U   n
U
 GH4  nUu  pUR                  R
                  nUR                  R
                  n/ nXS   ;   a  UR                  U5        / n[        [        U5      5       H  nUR                  U S	U 35        M     X;   a  SOS
nU	R                  5       R                  UR                  R
                     n[        UR                  5      nU(       a  SOUS   US'   X;   a  M  SSKJn  [        SUUU5        U" UUUUU/USSUUS
S
[        U	5      U5      nUS-  nUUUR!                  5       '   GM7     GML     [        U5       H  u  pU S   U   n[#        U	UUUUU5      nM     [        U S   5      S:  a   U S   (       a  [%        X`S   5      u  nnUUU'   U$ )Nr   r   r  zis_heter_ps_mode? r   Tr  r   .blockFr   z(public get_the_one_send_context sparse: r4   r   r  r	  )r   rl   r   rv   r   r   r~   ranger   rw   r   r   r   r   r   r   r
  r  r  )r  r   ep_listr   r   r  r   r  r  r   r  r   r  r   r   r  remote_sparse_idssplited_varnamer   r   r   r   r  r   r   r  s                             r-   get_the_one_send_contextr!  3  sA   #$HU<01J23O	01
23
C0$G0
#$9:1=)F KE,,I))..J "?33!((- O3w<(&&*VA3'?@ ) #:  &&(--doo.B.BCCOE*qaE!H$4:	 %7!J" 1HC.8HZ((*+e * 1l  0
"#78;$
 1 5 !A%%*<c#67	cOr/   c           
         U[         ;  a  [        SU S[          35      eSS jnSS jnS nU R                  5       nU R                  5       n 0 n[	        UR
                  5      n[        U5      n	[        U	S-
  SS5       GH  n
[	        UR
                  5      nX   nSUR                  ;   Ga  UR                  R                  S5      S	   nU[        R                  5       ;   Ga4  UR                  S
5      SL Ga  UR                  [        U   5      S	   nX;   a  [	        UR
                  5      nXU      nUR                  S	   UR                   Vs/ s H  nUR                   U   PM     sn0nUR"                  S	   UR$                   Vs/ s H  nUR                   U   PM     sn0nUR'                  U
S-   UR                  UUUR)                  5       S9  UR+                  X}   S-   5        UR-                  U5        U H  nUU==   S-  ss'   M     GM  GM  US:X  Gar   / nUR"                   HN  nUR/                  U5       H6  nUS:X  a  M  SU;   a  M  UR1                  UR                  S5      S	   5        M8     MP     U GH  nUU;   d  M   [	        UR
                  5      nXU      nUR                  S	   UR                   Vs/ s H  nUR                   U   PM     sn0nUR"                  S	   UR$                   Vs/ s H  nUR                   U   PM     sn0nUR'                  U
S-   UR                  UUUR)                  5       S9  UR+                  UU   S-   5        UR-                  U5        U H  nUU==   S-  ss'   M     GM
     GM  GM  UR                  S:X  d  GM(  UR/                  S5      S	   nSU;   d  GME  UR                  S5      S	   nXS-
     nSUR                  ;   Gad  UR                  R                  S5      S	   nU[        R                  5       ;   ac  UR                  S
5      SL aO  UR                  [        U   5      S	   nUU:X  a'  UR                  S5      UR                  S5      :X  a  GM  XU'   GM
  US:X  a  / nUR"                   H<  nUR/                  U5       H$  nUS:X  a  M  SU;   a  M  UR1                  U5        M&     M>     / nUR                   H<  nUR                  U5       H$  nUS:X  a  M  SU;   a  M  UR1                  U5        M&     M>     SnU H  nUU;   d  M  Sn  O   U(       a  GM  XU'   GM  GM  XU'   GM     U R                  5       nU R                  5       n/ nU0 0n0 n S	n!/ n"/ n#Un$Sn%UR
                   GH%  nU" UU$U5      (       a  Sn%[        U#5      S:  a   U#UU   U!'   UR1                  U#5        / n#U!S-  n!U" UU$U5      (       a  UR                  S5      n$U" UU"U 5        Mp  U"S	   R                  S5      n&U"U U&   U!'   UR1                  U"5        U!S-  n!/ n"UR                  S5      n$U" UU"U 5        M  U%(       aL  U"S	   R                  S5      n&U"U U&   U!'   UR1                  U"5        U!S-  n!/ n"Un$Sn%U#R1                  U5        GM  U#R1                  U5        GM(     U#/ :w  a  U#UU   U!'   UR1                  U#5        U"/ :w  a-  U"S	   R                  S5      n&U"U U&   U!'   UR1                  U"5        [        U 5      S	:X  a  [2        R4                  " S5        S	n'S	n(U R                  5        H>  n)U U)   n*U([        U*5      -  n(U*R7                  5        H  u  n+n,U'[        U,5      -  n'M     M@     [9        S[        UR
                  5       SU' SU( S35        UU UU4$ s  snf s  snf s  snf s  snf )NzGiven device z is not in device list c                     [        [        5      nUR                  U5        U R                  S5      nU R                  nXC;   a  gU[
        ;   a  X:w  a  gUb  XB:X  a  U R                  SU5        ggN	op_deviceTF)r   DEVICE_LISTremoverz   r   COMMUNICATE_OPS_TYPEr   )r   current_heter_devicedefault_deviceheter_devicesr%  op_types         r-   _is_heter_op$find_heter_ops.<locals>._is_heter_op  sm    [)^,GGK(	''%++$6
 )"=LLn5r/   c                 >    U R                  S5      nX1:X  a  gX:X  a  ggr$  )rz   )r   
pre_devicer*  r%  s       r-   _is_same_device'find_heter_ops.<locals>._is_same_device  s%    GGK(	"'r/   c                 Z    U R                  S5      nX2;  a  0 X#'   UR                  U 5        g )Nr%  )rz   r~   )r   current_heter_block_ops	heter_opsr%  s       r-   _append_heter_op(find_heter_ops.<locals>._append_heter_op  s,    GGK(	%#%I &&r*r/   r4   r  _gradr   remote_prefetchTr   r   inputsoutputsr  elementwise_mul@EMPTY@lod_tensor_blocking_queuer   sumOutr%  FzsNo heterogeneous OP was found in your program ,  please using static.device_guard() to run OPs on different device.z
There are z( OPs in your main_program, and contains z heter-OPs which is made up of z heter-blocks.r	   )r&  rk   clonerw   r   rx   r   r  r   r   SPARSE_OP_TYPE_DICTrj   rz   r   input_namesinput_arg_namesr   output_namesoutput_arg_names
_insert_opr   
_remove_oppopoutputr~   warningswarnitemsrl   )-r   r*  r-  r1  r6  origin_programr   var2idxop_listop_sizer  r   forward_op_typer  sum_opr   sum_op_inputsrL  sum_op_outputsvar_output_vars_no_gradrq   r   no_grad_varr   
origin_varpre_opoutput_vars
input_varsis_matchprogram_block_opsdefault_opsr5  block_indexr4  current_default_block_opsr)  is_heterr%  total_heter_opsheter_blocksdeviceheter_block_dict_heter_blocks-                                                r-   find_heter_opsrk    sl   [(N++B;-P
 	
(+ ]]_N  "E G599oG'lG7Q;B'uyy/Zbgg ggmmG4Q7O#6#;#;#==GG-.$6XX&9/&JKAN
("599oG$Z%89F**1-)/)?)?0)? "JJu-)?0%M ++A.*0*A*A1*A "JJv.*A1&N $$!e#[[, .$..0 %  $$W%81%<=KK
+ '* !(3 )6 !$55 ')#??C#%99S>"i/$6'A$+227==3I!3LM $2 + $7K"g- #'uyy/!()=!>"..q1-3-C-C4-CE !&

5 1-C4) #//2.4.E.E5.EF !&

6 2.E5* (("#a%!'#0$2"("2"2"4 )  (()=)ABK0$+D#DMQ.M %,; $7 6Z ww%ii&q)c>!$7!3A!6J$U^F&++-*0++*;*;G*DQ*G+/B/G/G/II &,= >$ F)/ 3O D**!J  *Z7BGG +=!'[!9=: !)67
 3,0AA*,K'-':':/5}}S/AG'.)';(0'Bg'M(0$/$6$6w$? 0B (; *,J')~~/1xx}G'.)';(0'Bg'M(0$.$5$5g$> 0= (6 (-H+6#*j#8/3H$) ,7  ( (67
 33 B6 /0
+C (F ]]_N  "E!2&KIK  ")Hii0.AAH ,-1- N+K8 "(()BC,.)q r#7HH')ww{';$ %<iH 4A6;;KH	4K	)$[1!(()@Aq *,'')ww{';$ %<iH/277DI0GIi -$$%<=1K&(##1 H%,,R0 &,,R0Q T !B&3LN#K0  !:;"$+A.33K@	,C	)[)  !89
9~R	

 OL.."$V,,--.446NA{s;//O 7 #
 

S^$$L_L]]|  ~J  }K  KY  	Z 9k3DDDI01L45s   ___
_
c                     [        U 5      n/ nUS-  S:w  d   S5       e[        SUS-  5       H5  nSX   0nUR                  SXS-
  U-
     05        UR                  U5        M7     / / S.nXS-      HM  nSUR                  ;  a&  UR                  S	:X  d  US   R                  U5        M9  US   R                  U5        MO     UR                  U5        U$ )
z
before analyzing the input & output of each block in program_block_list, we should
union the forward op and corresponding gradient op to eliminate the unnecessary variable
transmit
r5   r   z2the length of program_block_ops_list should be oddforwardbackwardr4   )rm  rn  r8  r@  )r   r  updater~   r   )program_block_ops_listblock_lengthunion_program_block_ops_listr  block_op_listr   s         r-   union_forward_gradient_oprt    s    -.L#% !q  <  1la'("$:$=>/q0@10DEF	
 	%++M: ) !#3M$Q%67"''!277e+;)$++B/*%,,R0	 8
 !''6''r/   c                 L    [        X5      n[        XX25      n[        XU5      nU$ ru   )find_entrance_exit_privateentrance_exit_checkdelete_block_useless_exit)r   rp  r5  block_var_details       r-   find_block_jointsrz    s>    1 +)9 1)9 r/   c                 :   / n/ nU Hg  n[        U R                  5       R                  U5      nU[        U5      -  n[	        U R                  5       R                  U5      nU[        U5      -  nMi     [        [        U5      5      n[        [        U5      5      nX#4$ ru   )_get_input_map_from_oprw   r   get_varlist_from_op_map_get_output_map_from_opr   r   )r   ops_listinput_var_listoutput_var_listr   r;  r<  s          r-   find_ops_list_input_outputr    s    NO'(<(<(>(C(CRH1&99)'*>*>*@*E*ErJ27;;	  #n-.N3/0O**r/   c           	         / n/ n[        U5       GHF  u  pE[        XS   5      u  pg[        X5      [        X5      -   n[        [	        U5      [	        U5      -  5      n[        [	        U5      [	        U5      -
  5      n	[        [	        U5      [	        U5      -
  5      n
SU	U
UUS.0n[        XS   5      u  p[        X5      [        X5      -   n[        [	        U5      [	        U5      -  5      n[        [	        U5      [	        U5      -
  5      n[        [	        U5      [	        U5      -
  5      nUR                  SUUUUS.05        UR                  U5        GMI     U$ )Nrm  )entranceexitprivatepersistablesrn  )rv   r  screen_persistablesr   r   ro  r~   )r   rp  ry  r  r   rs  block_inputblock_outputblock_private_varsblock_entrance
block_exitdetailbp_block_inputbp_block_outputbp_persistablesbp_block_private_varsbp_block_entrancebp_block_exits                     r-   rv  rv    s   L )*@ A$>9-%
! +
67 "#k"2S5F"FGc+.5G1HHI#l+c2D.EEF
*"- ,	
 +E:.+
' .
9: !%S%83;O%O P #&;"<<
 S1C8M4NNO 1)4$3			
 	'Y !BZ r/   c                 P   [        [        U5      S-
  SS5       GHH  nUS-
  S:  a    GO=X$S-
     S   S   nUR                  5         X$   S   S   nX$   S   S   nX$   S   S   X$   S   S   -   X$   S   S   -   nU H#  n	S	U	;  d  M  X;  d  M  UR                  U	5        M%     UR                  5         XV:X  a  M  [	        [        U5      [        U5      -  5      n
[	        [        U5      [        U
5      -
  5      nX$S-
     S   S   nX$S-
     S   S   nU HG  n	X;  a  X;  a  UR                  U	5        UR                  U	5        X;  d  M6  UR                  U	5        MI     GMK     [        S[        U5      S-
  S5       GH!  nX$S-      S   S   nUR                  5         X$   S   S   nUR                  5         XV:X  a  MB  [	        [        U5      [        U5      -  5      n
[	        [        U5      [        U
5      -
  5      n/ nU H  n	S	U	;  d  M  UR                  U	5        M     [	        [        U5      R                  [        U5      5      5      nX$S-      S   S   nX$S-      S   S   nU H/  n	X;  a  X;  a  UR                  U	5        UR                  U	5        M1     GM$     U$ )
Nr4   r  r   rm  r  r  rn  r  r   )r  r   sortr~   r   r   
difference)r   rp  ry  r5  r   previous_block_exitcurrent_block_entrancebackward_entranceforward_allr   
exist_varsneed_add_varsprevious_block_privateprevious_block_entranceneed_ignore_varss                  r-   rw  rw    s    s+,q0"b919q=.qy9)DVL  "!1!8!CJ!O,3J?
K #I.z:%i0;<%i089 	 %Cc!c&<&--c2 % 	##%8#$s+A'BB

 S!783z?JK
 "2!)!<Y!G"
 #319"=i"H#
 !C16'..s3&&s+0&--c2 !K :^ q#./!3Q7.qy9*EfM  "!1!8!DZ!P##%8#$s+A'BB

 S!783z?JK Cc! '', ! ))#.>*?@
 "2!)!<Z!H"
 #319"=j"I#
 !C16'..s3&&s+ !5 8B r/   c                    [        [        U5      5       Hk  nU[        U5      S-
  :X  a    OXX#   S   S   nX#S-      S   S   n/ nU H  nXu;  d  M
  UR                  U5        M     U H  nUR                  U5        M     Mm     [        [        U5      S-
  SS5       Hc  nUS-
  S:  a    U$ X#   S   S   nX#S-
     S   S   n/ nU H  nXu;  d  M
  UR                  U5        M     U H  nUR                  U5        M     Me     U$ )Nr4   rm  r  r  r  r   rn  )r  r   r~   r'  )r   rp  ry  r   current_block_exitnext_block_entranceneed_delete_varr   s           r-   rx  rx  R  sB    s+,-C()A---4Y?G.qy9)DZP%C-&&s+ & #C%%c* # . s+,q0"b919q=  .4Z@H.qy9*E
 %C-&&s+ & #C%%c* # : r/   c                 H   / n/ nUS:X  a  SUS-
   SU S3nOSUS-    SU S3nUR                  5         U Hb  nU R                  5       R                  U   nUR                  n	S[	        S U	S5      -  n
UR                  U
5        UR                  U S	35        Md     UUUS
.nU$ )Nrm  forward_joint_r4   ri  z@Heterbackward_joint_r  c                 
    X-  $ ru   r9   r   s     r-   r   *get_communicate_var_info.<locals>.<lambda>  s    r/   z.input_reshape@Heter)input_var_reshape_diminput_var_reshape_nameblock_input_var_name)r  rw   r   r   r   r~   )r   rb  entrance_var_listr   r  r  r  r   r   r   recv_var_diminfos               r-   get_communicate_var_infor  t  s     y[1_-Q{m6B 	
 kAo.a}FC 	  """$))$/		F#5ua@@$$\2%%.B&CD " "7"8 4D Kr/   c                 2   U  H  nXBR                  5       R                  ;  d  M"  XCR                  ;  d  M3  UR                  5       R                  U   nUR                  (       a   UR                  5       R                  USS9  M  UR                  USS9  M     g )NFforce_persistable)rw   r   persistable_clone_variable)var_name_listrP  r   r   r
  r   s         r-   add_vars_by_var_listr    s    !002777

* --/44X>C$$&665 7  %%cU%C "r/   c                    [         R                  " 5       nUR                   H\  n/ nUR                  U5       H&  nUS:X  a  M  SU;   a  M  UR	                  X   5        M(     [        U5      S:X  a	  US   X#'   MX  XBU'   M^     U$ )z9Returns a dict from op output name to the vars in varmap.r>  r?  r4   r   )collectionsOrderedDictrG  rL  r~   r   varmapr   iomaprq   r   r   s         r-   r~  r~    s    ##%Eyy~G)#*g5KK( & t9>aEJ#J  Lr/   c                     / nU R                  5        HW  u  p#[        U[        5      (       d  U/n[        [	        U5      5       H"  nX4   nUR                  UR                  5        M$     MY     U$ ru   )rO  
isinstancer   r  r   r~   r   )var_mapvar_listrq   varlistr  r   s         r-   r}  r}    s^    H'4((iGs7|$A*COOCHH% % ( Or/   c                    [         R                  " 5       nUR                   H\  n/ nUR                  U5       H&  nUS:X  a  M  SU;   a  M  UR	                  X   5        M(     [        U5      S:X  a	  US   X#'   MX  XBU'   M^     U$ )z8Returns a dict from op input name to the vars in varmap.r>  r?  r4   r   )r  r  rE  r   r~   r   r  s         r-   r|  r|    s    ##%E~~xx}G)#*g5KK( % t9>aEJ#J  Lr/   c                 t   / nU H  nSU;   aL  SUR                  S5      S   :w  a  M#  UR                  S5      S   nU R                  5       R                  U   nOU R                  5       R                  U   n[        U5      (       d  M  UR	                  U5        M     U H  nUR                  U5        M     U$ )Nr   GRAD@r  r   )r   rw   r   r   r~   r'  )r   r  need_remover
  origin_var_namer   s         r-   r  r    s    Kh,R00&nnW5a8O&&(--o>C&&(--h7C#x(   !  r/   c                     UR                  5       R                  R                  5       nUR                  UR                  5        [	        XC5      nUR                  5        H  u  pg[        U[        5      (       d  U/nU H  nUR                  U R                  5       R                  ;  d  M-  UR                  UR                  ;  d  MI  UR                  (       a   U R                  5       R                  USS9  Mz  UR                  USS9  M     M     [        UR                  5       R                  U5      n	U	R                  5        H  u  pg[        U[        5      (       d  U/nU H  nUR                  U R                  5       R                  ;  d  M-  UR                  UR                  ;  d  MI  UR                  (       a   U R                  5       R                  USS9  Mz  UR                  USS9  M     M     SUR                  ;  a)  UR                  UR                  XYUR                  5       S9$ UR                  n
[         R"                  R$                  R&                  n[         R"                  R)                  5       nUR                  R                  5       nUR+                  U
5        UR-                  [.        U5        UR                  R1                  U5      (       a"  U
R3                  U5      nUR-                  X5        UR5                  5         g )NFr  r8  r   r;  r<  r  )rw   r   copyro  r|  rO  r  r   r   r  r  r~  r   	append_opr   descr   r   r   r   kOpDeviceAttrName	copy_fromr   r{   has_attrrz   _sync_with_cpp)r   rP  r   r   merge_ordereddictr;  rq   r  r   r<  op_descrn  device_attr_namenew_op_descr%  s                  r-   block_append_opr    sk   &335::??AUZZ(#$5:F'4((iGC 4 4 6 ; ;;HHEJJ.??((*::u ;  ))#)G  ' &n&A&A&C&H&H"MG'4((iGC 4 4 6 ; ;;HHEJJ.??((*::u ;  ))#)G  ( bgg  
 	

 ''2299BB::LLN jj**,g&3X> 77,--%56I!!"2>r/   c                 d     U R                  5       $ ! [         a    U R                  5       s $ f = fru   )_get_next_trainersr   get_next_trainersr   s    r-   get_next_stage_trainersr  !  s4    .,,.. .++--.r   c                    U(       a0  [        U5      n[        U5      n	XS   S   S   n
[        XS-   U
5      nO3[        U5      n[        U5      n	XSS-
     S   S   n
[        XS-
  U
S5      nUR                  USSUR                  U
S      0S	/ 0S
U(       a  SOSS/ U
QSPS/ SUS   SUSU	S[        U5      SU[        [        0	S9  U
$ )Nrm  r  r4   rn  r  send_and_recvXr   rA  r   send_var_namemicrobatch_idrecv_var_namemessage_namer  next_endpointsprevious_endpointsr   r%  r:  )r  r   r  rI  r   r   r{   RPC_OP_ROLE_ATTR_VALUE)rP  r   rj  stage_idfirst_op_indexry  rg  
is_forwardnext_heter_worker_endpointsprevious_heter_worker_endpointsentrance_var	comm_infos               r-   insert_communicate_opr  (  s    &=j&I#*E+
' (1)<ZH,qL,
	
 '>j&I#*E+
' (15jA&I,qL,

	 [%%l1o67I=|=_=RI&<=9 "A+j1!#9

  $ r/   c                 ~   0 n0 nU(       a  [        XS9n[        UR                  5       5       H|  u  nu  pxUR                  5       (       a  M  UR	                  5       (       a  M6  UR                  5       n	/ n
U	 H  nU S   U   nU
R                  U5        M     XUR                  5       '   M~     U$ [        U SS S9n[        UR                  5       5       He  u  nu  pxUR                  5       (       d  M  UR                  5       n	/ n
U	 H  nU S   U   nU
R                  U5        M     XUR                  5       '   Mg     U$ )N)r   grad_name_to_param_nameF)r   r  )r!  rv   rO  r   is_tensor_tabler   r~   table_id)contextis_denser   recv_id_mapsr  r   r   r   r  origin_grad_varnamesparam_namesgrad_varnamer  s                r-   get_the_one_recv_contextr  [  sD   L +
 !*(..*: ;C$}}""$$#&#6#6#8 K 4$%>?M
"":. !5 ,7( !<8  ,ud
 !*(..*: ;C$==??#&#6#6#8 K 4$%>?M
"":. !5 ,7( !< r/   c                     SnSnSnU R                  S5      nUS:  a  XS-   S  nO[        U 5      nU R                  S5      nUS:  a  XS-   U nO[        U 5      nU S[        XT5       nXU4$ )Nrg   z	.trainer_r   r4   r  )findr   min)r   orig_var_nametrainer_part
block_parttrainer_idxrb  s         r-   _get_varname_partsr    s    MLJ,,{+KaQ01'l,,x(Ka1_{;
'lAK =>Ml22r/   r5   r7      r4   c                 b    [        S U R                  S5      nU[        U R                     -  nU$ )Nc                 
    X-  $ ru   r9   r   s     r-   r   "get_var_mem_size.<locals>.<lambda>  s    r/   r4   )r   r   dtype_to_sizedtype)r   m_sizes     r-   get_var_mem_sizer    s-    &		15F
mCII&&FMr/   c                       \ rS rSrS rSrg)MergedVariablei  c                 (    Xl         X l        X0l        g ru   )r   ordered_varsoffsets)r`   r   orderedr  s       r-   rd   MergedVariable.__init__  s     #r/   )r   r  r  N)r:   r;   r<   r=   rd   rD   r9   r/   r-   r	  r	    s    r/   r	  c                 h   U S   n0 n0 n/ U S'   / U S'   / U S'   / U S'   / U S'   0 U S'   U GH  n[        U5      u  pV/ n/ n/ n	/ n
/ nU H  u  pUR                  X45        M     U H  u  pUR                  X45        M     U HI  nUu  p[        X/S/5      n[        X/S/5      nUR                  UU45        U
R                  UU45        MK     U HI  nUu  p[        X/S/5      n[        X/S/5      nUR                  UU45        U	R                  UU45        MK     U HT  nUu  nnUR                  U S   UR                  R                  '   UR                  U S   UR                  R                  '   MV     / nUR                  U5        UR                  U5        U H5  u  pUR                  X,R                  '   UR                  X=R                  '   M7     U S   R                  U5        U S   R                  U5        U S   R                  U	5        U S   R                  U
5        GM     X S	'   X0S
'   g )Nr  origin_sparse_pairsorigin_dense_pairsr  r   merged_variables_pairsmerged_variable_mapr   param_name_to_grad_namer  )get_param_gradsr~   r	  r   r   extend)r  r  r  r  rP  sparse_pairsr   origin_for_sparseorigin_for_denser  r   r  r  r   
dense_pairm_paramm_gradsparse_pairr   param_mergess                       r-   build_var_distributedr    s   45O  %'G!"$&G !%'G!"$&G !(*G$%%'G!")$3N$C!  !#'KE$$e]3 ( 'KE##UM2 ' +J$KE$UGaS9G#D&1#6F"))7F*;<%%w&78 + -K%KE$UGaS9G#D&1#6F"))7F*;<&&'89 - -F$OGV"" )*7+=+=+B+BC !! )*6+<+<+A+AB - -.,-'KE26))#JJ/16#II. ( 	%&--.?@$%,,-=>%&--.AB$%,,-?@q *t *A%&)@%&r/   c                    [         R                  n[         R                  R                  R                  nUR	                  5       U R
                  ;   a8  [        U R                  5       UR	                  5          5      [        U5      :X  a  gg)NTF)r   r   r   OptimizekOpRoleAttrName
attr_namesry   r   )r   op_makeroptimize_roles      r-   r   r     sn     ..H33::CCM!R]]2s
x//128	]	8 r/   c                 D   ^  U 4S jnU 4S jnU" 5       nU" U5      u  pEXE4$ )Nc                   > TR                  5       n/ n/ n[        5       nTR                  5       R                  n[        [        R
                  R                  R                  5      nUR                   GH  n[        U5      (       d  M  [        UR                  5       ;   a1  [        UR                  [        5      ;   a  UR                  SU5        M_  UR                  [         5      (       d  M{  UR                  [         5      (       d  M  UR                  [         5      S   nUR                  [         5      S   n	X;  d  M  UR#                  U5        XX   XY   4n
X;   a  UR%                  U
5        M  UR%                  U
5        GM     X24$ )Nr   r   r4   )rw   r   r   ry   r   r   r   r   rx   r   r   r   r   rz   r   r  OP_ROLE_VAR_ATTR_NAMEr   r~   )sparse_varnamesr   dense_param_gradssparse_param_gradsoptimize_paramsorigin_var_dictr   r   r  r   
param_gradrP  s              r-   _get_params_grads*get_param_grads.<locals>._get_params_grads  s@   ++-%(557<<d55<<EEF))Br"" "R\\^3*bggm.DDLLG4{{#89977011!#)>!?!BJ "(= >q AI!8'++J7+7+6&

 &8.55jA-44Z@1 2 "44r/   c                  R  > / n TR                  5       R                   Hs  nUR                  [        R	                  5       ;   d  M'  UR                  S5      SL d  M=  UR                  [        UR                     5      S   nU R                  U5        Mu     [        [        U 5      5      $ )Nr9  Tr   )
rw   rx   r   rD  rj   rz   r   r~   r   r   )varnamesr   r  rP  s      r-   _get_sparse_varnames-get_param_grads.<locals>._get_sparse_varnames5  s     --/33B.3355GG-.$6XX&9"''&BCAF

+ 4 CM""r/   r9   )rP  r/  r3  r)  r+  r*  s   `     r-   r  r    s/    "5H
# +,O,=o,N)00r/   c                     U H9  n [        U R                  5      R                  U5      nU R                  U5        M;     g ! [         a  n[        U5         S nAMY  S nAff = fru   )r   rx   r   rJ  r   rl   )r   rx   r   r   es        r-   
delete_opsr7  G  sR    	uyy/''+CS!   	!HH	s   5A
A!AA!c                     / nU R                  5       R                   H&  nUR                  S:X  d  M  UR                  U5        M(     U$ )Nr   r   )r   send_op_listr   s      r-   find_send_opr:  P  sB    L""$((77f# ) r/   c                     / n/ n[        UR                  U5      nU[        U5      -  n[        UR                  U5      nU[        U5      -  n[	        [        U5      5      n[	        [        U5      5      nX44$ ru   )r|  r   r}  r~  r   r   )r   r   r   r  r  r;  r<  s          r-   find_op_input_outputr<  X  ss    NO#EJJ3F-f55N%ejj"5G.w77O#n-.N3/0O**r/   c                 n  ^  U 4S jn/ nU" 5       n0 nU H  nSU;  a  M  SUR                  S5      S   :w  a  M%  Xu;  a  M,  XW   nUR                  S5      n	UR                  S5      n
UR                  S5      nUR                  U5        X;  a  0 Xj'   / Xj   S	'   XU
   S'   XU
   S'   Xj   S	   R                  U5        M     U Hi  n
UR                  [	        5       S
9nXj   S	    Vs/ s H  nUR
                  U   PM     nnUR                  SSU0SU0SXj   S   SW	SU
[        [        0S9  Mk     U$ s  snf )Nc                     > 0 n [        T5      nU H-  n[        TTR                  5       U5      u  p4U H  nX U'   M	     M/     U $ ru   )r:  r<  rw   )send_op_dictr9  r   
input_listri  r   r   s         r-   _get_send_op_dict&add_send_op.<locals>._get_send_op_dicte  sT    #G,B0--/MJ "$&S! "	  r/   r   r  r  r  r   r  send_varnamesr  )r   r   r  rA  r  )	r   rz   r~   
create_varr   r   r  r{   r  )r   r   _varsrA  send_grad_var_listr?  
table_dictpersistable_varsend_opr   r  rC  dummy_output	union_varsend_input_varss   `              r-   add_send_oprM  d  s   	 $&LJ /)_**3/33./LL-	<<
+_5!!/2%#%J /1J ,09x -4Ax 1Z(//@# !& ''-J-L'M (1*=
=	 JJy!= 	 
 	)L)!5o!FYH%'=		 	 
	
 $ !
s   &D2c                 P    U R                   R                  5       n[        U5      nU$ ru   )r   rj   r   )r   	vars_listvars_name_lists      r-   get_vars_name_in_blockrQ    s!    

!I)_Nr/   c                 :   [        [        U5      5      n/ nU R                  5       R                   Hw  n[	        X R                  5       U5      u  pE[        [        U5      R                  [        U5      5      5      n[        [        U5      R                  [        U5      5      5      nMy     X!-  n[        [        [        U R                  5       5      5      R                  [        U5      5      5      nU H"  nU R                  5       R                  U5        M$     U$ ru   )	r   r   rw   rx   r<  unionrQ  r  _remove_var)	r   
static_varprogram_useful_var_listr   r  r  op_var_listprogram_useless_var_listr   s	            r-   delete_trainer_useless_varrY    s    c*o&J ""$((*>))+R+
' 3~.44S5IJK"&'(..s;/?@#
 ) )#"7#7#7#9:;FF'(	
 
 (**3/ (##r/   c                    U R                   S-
  nU R                  U5      n[        U5       Hx  u  pgUR                  S:X  aW  UR	                  S5      nSn	U H5  n
XR                  5       R                  ;  d  M"  XR                  ;  d  M3  Sn	  O   U	(       a  Ml  [        XXW5        Mz     US   S   S   n[        XX5        US   S   S	   n[        XX5        U$ )
Nr4   r   rC  FTr   rn  r  r  )	
num_blocks_create_blockrv   r   rz   rw   r   r  r  )r   rP  bp_ops_listry  pre_block_idxrj  ri  r   rC  is_skipr   entrance_vars	exit_varss                r-   create_backward_blockrb    s     &&*M''6K;'77fGGO4MG(#7#7#9#>#>>'7'77"G ) A ( %Q'
3J?MM #J/7IGIr/   c                     [         U R                  ;   =(       a8    [        U R                  [         5      5      [        [        R
                  5      -  $ ru   )op_role_attr_namer#  ry   rz   r   r   r   s    r-   is_backward_opre    s:    - BGG%&'#g.>.>*??r/   c                     [         U R                  ;   =(       a8    [        U R                  [         5      5      [        [        R
                  5      :H  $ ru   )rd  r#  ry   rz   r   Forwardr   s    r-   is_forward_oprh    s8    - BGG%&'3w+??r/   c                      U R                   S:H  $ )Ndistributed_push_sparse)r   r   s    r-   is_push_sparse_oprk    s    77///r/   c                     / n[        U R                  R                  5       5       H5  nU R                  U   n[	        U5      (       d  M$  UR                  U5        M7     U$ ru   )r  r  rS  rx   rk  r~   )r   push_sparse_op_listop_idxr   s       r-   #get_distributed_push_sparse_op_listro    sS    

**,-YYvR  &&r* . r/   c                     / n[        U R                  R                  5       5       H5  nU R                  U   n[	        U5      (       d  M$  UR                  U5        M7     U$ ru   )r  r  rS  rx   re  r~   )r   
bp_op_listrn  r   s       r-   get_bp_op_listrr    sR    J

**,-YYv"b! . r/   c                 $   U Hg  n U R                    HS  n[        U5      [        U5      :X  d  M  [        U R                   5      R                  U5      nU R	                  U5          Me     Mi     g ! [
         a  n[        U5         S nAM  S nAff = fru   )rx   rm   r   r   rJ  r   rl   )r   rx   r   	origin_opr   r6  s         r-   delete_same_opsru    ss    	"YY	y>SW,uyy///	:C$$S)	 '   	!HH	s"   %A/6A/(A//
B9B

Bc           	      R   SnU R                    H  nUR                   Hr  nUR                  R                  5       nUR                  R	                  5       nXE-    H3  nUR                  [        U5      5      (       a  M$  [        SU SU 35      e   Mt     US-  nM     [        S5        g )Nr   zvar: z% needed by op is not found in block: r4   zprogram checked valid)	blocksrx   r  rF  rH  _find_var_recursiverm   rk   rl   )r   	block_idxr   r   input_var_namesoutput_var_namesr
  s          r-   check_programr|    s    I))B gg557O!ww779+>00X??$z)NykZ  ?  	Q	   

!"r/   c                     [         R                  " [         R                  R                  U 5      SS9  [	        U S5       nUR                  [        U5      5        S S S 5        g ! , (       d  f       g = f)NT)exist_okzw+)r[   makedirspathdirnameopenwriterm   )filer   fs      r-   debug_programr    sE    KK%5	dD	Q	G 
		s   A##
A1c                  8    [         R                  " S5      n U c  gg)NTRAINING_ROLEFT)r[   r\   )	node_roles    r-   is_distributed_envr    s    		/*Ir/   )F)FNrB  )rm  )T)TF)ur  r   r[   rM  	functoolsr   paddle.base.frameworkr   paddle.distributed.ior   paddle.frameworkr   r   r   r  LEARNING_RATE_DECAY_COUNTERr   kOpRoleVarAttrNamer(  r"  r{   r   RPCr  r   rd  LRSchedr|   r!  r}   r   rn  r  OP_DEVICE_KEYr&  r(  r   rD  SPARSE_GRAD_OP_TYPE_DICTDEFAULT_DEVICEDATA_NORM_NAMEr   r.   ps_log_root_dirr)   r2   rF   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r!  rk  rt  rz  r  rv  rw  rx  r  r  r~  r}  r|  r  r  r  r  r  r  VarDescVarTypeFP16FP32FP64INT16INT32INT64BOOLUINT8r  r  r	  r  r   r  r7  r:  r<  rM  rQ  rY  rb  re  rh  rk  ro  rr  ru  r|  r  r  )r   s   0r-   <module>r     s     	   ? 0 ! $ "2 77JJL 77GGI 88??CC 

)
)
0
033CCE "==DDLL 88??HH **11:://AAC#H  "34'*sC   C,:;Nq7{N; $ 	!0B

 fJ fJR .0 0$$K23./2
"' yx8v6PfDEN(@+0fSlF 3<DD$$&7t. 0f#L3( 	LLqLLqLLqLLLLLLLLqLL	 Tn	41n	+4n$,80	#C7 <s   M