
    x-j                     b    d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	  G d de          Z
dS )    N)LauncherInterface)loggerpull_worker_logstart_local_trainersc                   &    e Zd Zd Zd Zd Zd ZdS )CollectiveLauncherc                 "    || _         g | _        d S )N)argsprocs)selfr
   s     k/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/fleet/elastic/collective.py__init__zCollectiveLauncher.__init__   s    	


    c                    t          j        d           | j        }t          j                    | _        t          j        j        j	        
                    |          \  }}t          j        j        j	                            || j                  }t          |||j        |j        |j        |          | _        t#          | j                  D ])\  }}t          j        d|j        j         d|            *d S )Nzcollective launcher launch ...)training_scripttraining_script_argslog_direnvszlaunch proc_id:z idx:)r   infor
   tempfilemkdtemptmp_dirpaddledistributedfleetlaunchget_cluster_infoget_global_envsr   r   r   r   r   	enumerateprocpid)r   r
   clusterpodglobal_envsidxr    s          r   r   zCollectiveLauncher.launch!   s    4555y')))/6GGMM(.5EE$,
 
 * 0!%!:L
 
 

 #4:.. 	E 	EICKC$)-CCcCCDDDD	E 	Er   c                     t          j        d           |                                 st          j        d           t          j                            | j                  rt          j	        | j                   d S d S )Nzcollective launcher stop ...zkill process failed)
r   r   _terminate_procserrorospathexistsr   shutilrmtree)r   s    r   stopzCollectiveLauncher.stop6   sr    2333$$&& 	0L.///7>>$,'' 	(M$,'''''	( 	(r   c                     t          j        d           | j        D ]#}|j        r|j        dk    rt          |           $|                                 }|S )Nzcollective launcher watch ...r   )r   debugr   log_fn
local_rankr   _check_procs)r   prets      r   watchzCollectiveLauncher.watch=   s_    4555 	# 	#Ax #ALA--"""!!
r   N)__name__
__module____qualname__r   r   r.   r6    r   r   r   r      sS          E E E*( ( (    r   r   )r)   r,   r   r   (paddle.distributed.fleet.elastic.managerr   %paddle.distributed.fleet.launch_utilsr   r   r   r   r:   r   r   <module>r=      s    
			    F F F F F F         ' ' ' ' '* ' ' ' ' 'r   