
    Αi                     P    S SK r S SKrS SKJr  S SKrSSKJrJrJr   " S S5      r	g)    N)Thread   )get_gpu_infoget_gpu_processget_gpu_utilc                   2    \ rS rSrS rS rS rS rS rSr	g)	Watcher   c                 Z   Xl         SU l        / U l        U R                   R                  R                  (       d  g [
        R                  " 5       (       a  g U R                   R                  R                  =(       d*    U R                   R                  R                  R                  U l        [        U R                  5      S:  a  [        R                  R                  U R                   R                  R                   U R                   R                  R"                   S35      n[        R$                  " [        R                  R'                  U5      SS9  [)        US5      U l        Og [-        U R.                  S9U l        SU R0                  l        U R0                  R5                  5         g )N   r   z.gpu.logT)exist_okw)target)ctxintervalgpu_utilargsenable_gpu_logpaddleis_compiled_with_rocmdevicesnodedevicelabelsgpuslenospathjoinlog_dirjob_idmakedirsdirnameopengpu_fdr   watchprocdaemonstart)selfr   fns      m/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/launch/controllers/watcher.py__init__Watcher.__init__   s   xx}}++'')) HHMM))HTXX]]-A-A-H-H	tyy>A%%$((--*>*>)?x'HB KK+d;r3-DK $**-					    c                 *   [        U R                  5      S:  d  g U R                  5         SnU R                  R	                  U5        U R                  R	                  S5        U R
                  R                  R                  5       (       d\  U R                  U5        [        R                  " U R                  5        U R
                  R                  R                  5       (       d  M\  [        U S5      (       a  U R                  R                  5         g g )Nr   zDindex,utilization_gpu,memory_total,memory_used,memory_free,timestamp
r%   )r   r   _print_gpu_infor%   writer   statusis_done_save_gpu_logtimesleepr   hasattrclose)r*   util_keys     r,   r&   Watcher.watch6   s    499~!Y(#$((//))++x(JJt}}% ((//))++ 4""KK #r/   c                     SnU R                   R                  U5        U R                   R                  S5        [        U R                  5       HH  nU R                   R                  UR	                  U5      5        U R                   R                  S5        MJ     U R                   R                  S5        SnU R                   R                  U5        U R                   R                  S5        [        U R                  5       HH  nU R                   R                  UR	                  U5      5        U R                   R                  S5        MJ     U R                   R                  S5        U R                   R                  5         g ! [         a3  nU R                  R                  R                  SU< 35         S nAg S nAff = f)NzEindex,uuid,driver_version,name,gpu_serial,display_active,display_moder1   z.pid,process_name,gpu_uuid,gpu_name,used_memoryzsave gpu info failed: )r%   r3   r   r   strr   flush	Exceptionr   loggerwarning)r*   info_keylineprocess_keyes        r,   r2   Watcher._print_gpu_infoG   sE   	D^HKKh'KKd#$TYY/!!$((8"45!!$' 0 KKd#JKKKk*KKd#'		2!!$((;"78!!$' 3 KKd#KK 	DHHOO##&<QE$BCC	Ds   FF 
G)F==Gc                 |    [        U R                  5       HH  nU R                  R                  UR	                  U5      5        U R                  R                  S5        MJ     U R                  R                  5         g ! [         a3  nU R                  R                  R                  SU< 35         S nAg S nAff = f)Nr1   zsave gpu log failed: )
r   r   r%   r3   r>   r?   r@   r   rA   rB   )r*   r;   rD   rF   s       r,   r6   Watcher._save_gpu_log]   s    	C$TYY/!!$((8"45!!$' 0 KK 	CHHOO##&;A5$ABB	Cs   A;A> >
B;)B66B;c                 (    [        U S5      (       a  g g )Nr'   )r9   )r*   s    r,   stopWatcher.stopf   s    4    !r/   )r   r%   r   r   r   r'   N)
__name__
__module____qualname____firstlineno__r-   r&   r2   r6   rK   __static_attributes__ r/   r,   r	   r	      s    : "D,Cr/   r	   )
r   r7   	threadingr   r   utils.nvsmir   r   r   r	   rR   r/   r,   <module>rU      s%    
    E ER Rr/   