
    x-j                         d dl Z d dlmZ d dlmZ ddlmZ g Zej        j	        Z	ej        
                                Zej                                        Zd Zd Zd Zd	 Z G d
 d          ZdS )    N)core)unique_name   )wait_server_readyc                 8    d| j         v od| j         v od| j         v S )NParamGradLearningRate)input_namesops    o/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/fleet/meta_optimizers/common.pyis_update_opr      s/    2>! 	-bn$	-bn,    c                     t           | j        vrdS t          |                                 t                              }|t          t          j                  z  o|t          t          j                  z  S )NF)OP_ROLE_KEY
attr_namesint	all_attrsOpRoleBackwardLoss)r   op_roles     r   is_loss_grad_opr   &   sW    "-''u",,..-..GS)))HgFK8H8H.HHr   c                     t           | j        v oEt          |                                 t                              t          t          j                  z  S N)r   r   r   r   r   r   r   s    r   is_backward_opr   -   D    "-' C
{#- -FO- r   c                     t           | j        v oEt          |                                 t                              t          t          j                  z  S r   )r   r   r   r   r   Optimizer   s    r   is_optimizer_opr!   3   r   r   c                   6    e Zd Zd	dZd
dZ	 	 ddZd Zd ZdS )CollectiveHelper   Tc                 0    || _         || _        || _        d S r   )nrings	wait_port
role_maker)selfr(   r&   r'   s       r   __init__zCollectiveHelper.__init__:   s    "$r   Nc           	         || _         |#t          j                                        | _         | j                                        }|| j                                                 }t          | j                  D ]=}| 	                    | j         ||| j                                        || j
                   >|                                  d S r   )startup_programpaddlestaticdefault_startup_programr(   _get_trainer_endpoints_worker_indexranger&   _init_communicatorr'   _broadcast_params)r)   r,   	endpointscurrent_endpointring_ids        r   update_startup_programz'CollectiveHelper.update_startup_program?   s    ."#)=#H#H#J#JD O::<<	$T_%B%B%D%DET[)) 	 	G##$ --//    	     r   c	                    d                     |          }	|r.t          |          }
|d d          }|                    |           fd}|                                }||J |sJ  ||           d S |                    t          j        d          dt          j        j	        j
                  }t          j                    ri|                    di d|id|d	|d
|d|t          t          j        i           |                    dd|ii d|
d|d|d|	t          t          j        i           nt          j                    ri|                    di d|id|d	|d
|d|t          t          j        i           |                    dd|ii d|
d|d|d|	t          t          j        i           nt#          d          |r ||           d S d S )N,c                    |                      t          j        d          t          j        j        j        dd          }|                     di d|giddgd	|j        d
dddt          t          j        i           |                     dd|gid|giddt          j        j        j        t          t          j        i           |                     dd|id|it          t          j        i           d S )Nsync_varFT)namedtypepersistablestop_gradientfill_constantOutshaper$   r>   value	force_cputypeinputsoutputsattrs
all_reducexoutr7   reduce_typec_sync_calc_streamX)
create_varr   generater   VarDescVarTypeINT32	append_opr>   r   r   Forwardr-   distributedReduceOpSUM)blockr<   global_ring_ids     r   _add_sync_by_allreducezCCollectiveHelper._init_communicator.<locals>._add_sync_by_allreduced   s#   '' )*55l*0!"	 (  H OO$
+aSX^Q	     OO!hZ(
+~!6#5#>#B	  	 	 	 OO)X)"FN3	      r   comm_idT)r=   r?   rG   c_gen_nccl_idrB   rankendpointother_endpointsr7   rF   c_comm_initrP   nranksr5   c_gen_bkcl_idzBcomm_id must be generated in paddlepaddle-xpu or paddlepaddle-xpu.)joinlenremoveglobal_blockrQ   r   rR   r   rS   rT   RAWis_compiled_with_cudarV   r   r   rW   is_compiled_with_xpu
ValueError)r)   programr6   r5   r`   r7   r'   r\   syncendpoints_strrd   rb   r]   r[   comm_id_vars          `       r   r3   z#CollectiveHelper._init_communicatorQ   s{    ++ 	5^^F'lO""#3444"	 "	 "	 "	 "	H $$&&#$$$KK4""5)))F&&%i00%) ' 
 

 %'' 5	OO$,D 0%w	     OO"[)fDw	      &(( 	OO$,D 0%w	     OO"[)fDw	      T    	*""5)))))	* 	*r   c                 t    | j         sJ |d d          }|                    |           t          |           d S r   )r'   rh   r   )r)   r6   r5   rb   s       r   _waitzCollectiveHelper._wait   sE    ~~#AAA,/000/*****r   c                    | j                                         }d}|                                D ]I}|j        r
|dz   | j        z  }|                    dd|id|id|ddt          t          j        i	           Jt          | j                  D ]2}|                    d
d|id|id|t          t          j        i	           3d S )Nr$   	broadcastrL   rM   r7   rootr   rF   c_sync_comm_streamrP   rB   )
r,   ri   iter_parametersis_distributedr&   rV   r   r   rW   r2   )r)   r[   r7   params       r   r4   z"CollectiveHelper._broadcast_params   s    $1133**,, 	 	E# {dk1GOO U|wA	  	 	 	 	 T[)) 	 	GOO)U| ';G	     	 	r   )r$   Tr   )NT)__name__
__module____qualname__r*   r8   r3   rs   r4    r   r   r#   r#   9   s{        % % % %
! ! ! !4 z* z* z* z*x+ + +    r   r#   )r-   paddle.frameworkr   paddle.utilsr   base.private_helper_functionr   __all__op_proto_and_checker_makerr   kOpRoleAttrNamer   kOpRoleVarAttrNameOP_ROLE_VAR_KEYr   r   r   r!   r#   r   r   r   <module>r      s      ! ! ! ! ! ! $ $ $ $ $ $ < < < < < <
		(	/-==??1DDFF  I I I    s s s s s s s s s sr   