
    x-j/<                        d Z ddlmZ ddlZddlZddlmZ ddlZerddlmZ ddl	m
Z
 ddlmZmZ g Z G d d	          ZdS )
zParameter Server utils    )annotationsN)TYPE_CHECKING)Tensor)RoleMakerBase)ExecutorProgramc                  J    e Zd ZdZ	 	 dddZ	 	 dddZd ZddZddZd Z	dS )DistributedInferz>
    Utility class for distributed infer of PaddlePaddle.
    Nmain_programProgram | Nonestartup_programreturnNonec                   |r|                                 | _        n5t          j                                                                         | _        |r|| _        n#t          j                                        | _        d | _        d S N)cloneorigin_main_programpaddlestaticdefault_main_programorigin_startup_programdefault_startup_programsparse_table_maps)selfr   r   s      f/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/fleet/utils/ps_util.py__init__zDistributedInfer.__init__%   s    
  	'3'9'9';';D$$ 2244::<< $  	*9D'' 5577 ' "&    exer   lossr   
role_makerRoleMakerBase | Nonedirname
str | Nonec                   ddl m} |j        j        h|                    |           t          j                                        }|                                }d|_        |	                    ||          }|
                    || j                   |                                r+|                    |           |                                 n[|                    t          j                                                   |                                 |                     ||           t          j                                        }	| j        }	t          j                                        }
| j        }
d S d S )Nr   fleet)r    T)strategy)r   )r"   )paddle.distributedr&   _runtime_handleinitr   	optimizerSGDDistributedStrategya_syncdistributed_optimizerminimizer   	is_serverinit_server
run_serverrunr   r   init_worker_init_dense_paramsr   r   )r   r   r   r    r"   r&   fake_optimizerr'   r+   global_startup_programglobal_main_programs              r   init_distributed_infer_envz+DistributedInfer.init_distributed_infer_env9   sw    	-,,,,,;&.JJ*J---#-1133N0022H"HO33 4  I d&A        6!!'!222  """"==??@@@!!###''W555%+]%J%J%L%L"%)%@""(-"D"D"F"F"&":/ /.r   c                    ddl m} | j        ki | _        |j        j        j        }|                                D ]>\  }}|j        r2|                    d          }|                                | j        |<   >?| j        S )Nr   r%   z@GRAD)	r(   r&   r   r)   	_send_ctxitems	is_sparsestriptable_id)r   r&   send_ctxgradnamectxparams         r   _get_sparse_table_mapz&DistributedInfer._get_sparse_table_map[   s    ,,,,,,!)%'D"{2<H!)!1!1  #= $NN733E47LLNND*511%%r   c                   |                                  k|kd | j                                        D             }fd|D             }fd|D             }t          j                            || j        |           d S d S d S )Nc                Z    g | ](}t           j        j                            |          &|)S  )r   r   iois_persistable).0vs     r   
<listcomp>z7DistributedInfer._init_dense_params.<locals>.<listcomp>m   sA          =#22155      r   c                4    g | ]}|j         v|j         |fS rH   )name)rK   rL   r   s     r   rM   z7DistributedInfer._init_dense_params.<locals>.<listcomp>r   s7     " " "6!222 222r   c           	         g | ]M}t           j                            t           j                            |d                              E|d         NS )r      )ospathisfilejoin)rK   rL   r"   s     r   rM   z7DistributedInfer._init_dense_params.<locals>.<listcomp>w   sU       7>>"',,w!"="=>>!  r   )r   vars)rE   r   	list_varsr   r   	load_vars)r   r   r"   all_persist_varsdense_persist_varsneed_load_varsr   s     `   @r   r6   z#DistributedInfer._init_dense_paramsi   s     66883?   1;;==     
" " " ")" " "
   +  N
 M##!5#	 $     ! ??r   r   c                d    |                                  }|                     | j        |          }|S r   )rE   _convert_programr   )r   varname2tablesconvert_programs      r   get_dist_infer_programz'DistributedInfer.get_dist_infer_program   s8    3355//$n
 
 r   c                (    fd} ||          }|S )Nc                     dddfd} fd} |           }t          j        d            | |            S )NW)lookup_tablelookup_table_v2c                T   i }|                                  j        D ]}|j                                        v rm|                    d          du rV|                    |j                           d         }|                    |g           }|                    |           |||<   |S )Nremote_prefetchTr   )global_blockopstypekeysattrinputgetappend)_programpull_sparse_opsop
param_nameri   SPARSE_OP_TYPE_DICTs        r   _get_pull_sparse_opsz]DistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._get_pull_sparse_ops   s    "$"//115 : :B#6#;#;#=#===GG$566$>>%'XX.A"'.J%K%KA%N
-11*bAA

269
3&&r   c                   d }|                                 D ]8\  }}                                j        fd|D             }                                j        |d                             d          d                  }|j                                        vrt          d|j         d          |j                 }|d                             d          }|d                             d          }	|d         j	        }
fd	|D             } |||           fd
|D             }|d d d         D ])}                                
                    |           *dgt          |          z  }t                                          j                  dz   gt          |          z  }t                                          j                  D ]\  }}t          dt          |j                            D ]Y}|                    |j        |                   }t          |          D ]'\  }}|j        |v rt!          |||                   ||<   (Zt          dt          |j                            D ]Y}|                    |j        |                   }t          |          D ]'\  }}|j        |v rt%          |||                   ||<   (Zt%          |          t!          |          z
  dk    rKt!          |          dz   }                                                    |d||dd|i|	||d|
d           +t          d          d S )Nc                   |                                  }t          |j                  }d}dgt          |j                  z  }dgt          |j                  z  }t          |j                  D ]\  }}	t	          dt          |	j                            D ]e}
||         dk    r nV|	                    |	j        |
                   }t          |          D ]%\  }}|j        |v rd||<   t          ||          } n&ft	          dt          |	j	                            D ]c}
||         dk    r nT|	
                    |	j	        |
                   }t          |          D ]#\  }}|j        |v rd||<   t          ||          }$dt	          t          |j                            D ]1}
||
         dk    r#||
         dk    rt          j        d            d S 2||k     rg }t	          |dz   t          |                    D ]0}
||
         dk    r"|                    |j        |
         |
f           1t          |          D ]\  }
}	g }t                      }|                    |	d                    |                    |	d                    d}|t          |          k     r||         }|j        |         }	g }t	          dt          |	j	                            D ]7}|	
                    |	j	        |                   }|                    |           8t	          |dz
  |dz
  d          D ]}|j        |         }||v rd}t	          dt          |j                            D ]\}|                    |j        |                   }t	          t          |                    D ]}||         D ]
}||v rd} n|r n|r n]|rU||         rt          j        d             d S |                    |           |                    |j        |                    |dz   }|t          |          k     |                                 |D ]}|j                            |          }|                    |j        |         j                   |j                            |dz   |dz              ||j        |         _        |j                            |          }|                    |          } |                    |          }!|j                            ||           |                    ||            |                    ||!           |dz   }|j                                        t          |j                  k    sJ t	          t          |j                            D ]4}
|j                            |
          |j        |
         j        k    sJ 3d S d S )	Nr   rQ   zunable to re-arrange dags order to combine distributed embedding ops because a op both needs embedding table's output as input and produces ids as the same embedding table's inputFTzDunable to re-arrange dags order to combine distributed embedding ops   )rh   lenri   	enumeraterangeoutput_namesoutputrO   maxinput_namesrm   minwarningswarnro   setaddsortdesc
_insert_op	copy_from
_remove_oppopinsertop_sizerr   )"programinputsoutputsrh   min_output_indexmax_input_indexinput_indexesoutput_indexesidxrr   ioutsin_idin_varinsout_idout_varmove_opsqueuevisitedstartpos	op_inputskjop1foundtyindexr   	insert_opinput_stateoutput_states"                                     r   dag_check_up_and_reorderz|DistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._pull_sparse_fuse.<locals>.dag_check_up_and_reorder   s   #*#7#7#9#9L'*<+;'<'<$&(O%&C#l.>*?*?$?M&'S3|/?+@+@%@N#,\-=#>#> & &R!&q#bo*>*>!?!? * *A,S1Q66 %#%99R_Q-?#@#@D1:61B1B * *v#);$#6#69:M#$669/36O6OO$)E $7
 "'q#bn*=*=!>!> 	& 	&A-c2a77 %"$((2>!+<"="=C3<W3E3E & &#*<3#6#6:;N3$77:(8#8& 8&$4& #3|'7#8#899 # #(+q00^A5F!5K5K$M !V   #FF'/99#%!&,q0#m2D2D" " J JA  -Q/144 (1A!1Da0H I I I%.x%8%8 ?H ?HEAr$&E&)eeG!LLA///#KK1...$%E"'#e**"4"4&+El%1%5c%:,.	).q#bn2E2E)F)F !: !:A*,((2>!3D*E*EC$-$4$4S$9$9$9$9).$'!G-=-A2*" *" !I !IA +7*:1*=C'*g~~(0,1E-21c#:J6K6K-L-L 
%2 
%2/2zz#:J1:M/N/N16s9~~1F1F )6 )6A5>q\ -: -:3499<@E49E 4= 05 -605-6+0 )2,1E)2', %I+9!+< )3,4M0v-. -. -. -3FFF(-Q(/L4DQ4G(H(H(H(-	C #(#e**"4"4F "JJLLL). H H'3'8'C'C$4(" (" !%|/?/F/K L L L , 1 < <$)AIuqy!" !" !" @D 0 7 <,8,<,@,@,G,G	.;.?.?.F.F/=/A/A%/H/H , 0 7 7$4i!" !" !" !. 4 4$4k!" !" !" !/ 5 5$4l!" !" !" 4Da3G 0 0+H.  ,088::c(,? ?         "'s<+;'<'<!=!=  A , 1 4 4Q 7 7#/#3A#6#;!< !< !< !< !<[ :9V r   c                    g | ]:}                                 j        |                    d           d                  ;S )Idsr   )rh   rV   rm   rK   rr   r   s     r   rM   znDistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._pull_sparse_fuse.<locals>.<listcomp>  sL         ,,..3BHHUOOA4FG  r   r   rc   zcan not find variable z!, please check your configurationpadding_idxis_distributedc                    g | ]:}                                 j        |                    d           d                  ;S )Outr   )rh   rV   r~   r   s     r   rM   znDistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._pull_sparse_fuse.<locals>.<listcomp>%  sN         ,,..3BIIe4D4DQ4GH  r   c                :    g | ]}                     |          S rH   )r   )rK   rr   all_opss     r   rM   znDistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._pull_sparse_fuse.<locals>.<listcomp>+  s%    ???Rw}}R00???r   rx   rQ   distributed_lookup_table)r   rc   OutputsT)r   r   r@   is_testlookup_table_version)r   rj   r   r   attrsz9something wrong with Fleet, submit a issue is recommended)r=   rh   ri   rV   rm   rO   rk   
ValueErrorrl   rj   r   rz   r{   r|   r}   r~   r   r   r   r   )rp   rq   r   rD   ri   r   wr@   r   r   op_typer   op_idxsr   inputs_idxsoutputs_idxsrr   r   r   r   r   r   r   r   distributed_idxr   r   r^   s                            @r   _pull_sparse_fusezZDistributedInfer._convert_program.<locals>.distributed_ops_pass.<locals>._pull_sparse_fuse   s   r r rh #2"7"7"9"9 H HJE3%22448G   "%  F
  ,,..3CFLL4E4Ea4HIAv^%8%8%:%:::(^QV^^^    .af5H"%a&++m"<"<K%(V[[1A%B%BN!!fkG   "%  G
 -,WfgFFF????3???G&ttt} ? ?,,..99#>>>>#%$V"4K$'(<(<(>(>(B$C$Ca$G#H3L L $L $-W-A-A-C-C-G#H#H & &R!&q#bo*>*>!?!? & &A#%99R_Q-?#@#@D1:61B1B & &v#);$#6#69<(+[-?:& :&K$6&
 "'q#bn*=*=!>!> & &A"$((2>!+<"="=C3<W3E3E & &#*<3#6#6;>(+\&-A<& <&L$8&& <((3{+;+;;q@@*-k*:*:Q*>,,..99"1!;+1#:#:%.$82@/:,4+/8?# # :     )W  MH Hr   zBlookup_table will be forced to test mode when use DistributedInfer)r   r   )r   ru   r   rq   rt   r^   s   `   @r   distributed_ops_passz?DistributedInfer._convert_program.<locals>.distributed_ops_pass   s    363"O"O' ' ' ' '} } } } } }~ 327;;OMT   g777Nr   rH   )r   r   r^   r   covert_programs     `  r   r]   z!DistributedInfer._convert_program   s:    T	 T	 T	 T	 T	l .-l;;r   )NN)r   r   r   r   r   r   )
r   r   r   r   r    r!   r"   r#   r   r   )r   r   )
__name__
__module____qualname____doc__r   r:   rE   r6   r`   r]   rH   r   r   r
   r
       s          (,*.& & & & &0 ,0" ;  ;  ;  ;  ;D& & &   4   X X X X Xr   r
   )r   
__future__r   rR   r   typingr   r   r   (paddle.distributed.fleet.base.role_makerr   paddle.staticr   r   __all__r
   rH   r   r   <module>r      s      " " " " " " 				               0FFFFFF////////
B B B B B B B B B Br   