
    x-j/                     N    d dl Z d dlZd dlZd dlZd dlZd Z G d d          ZdS )    Nc                     t          j        | t           j        t           j        dd          }|                                \  }}|rt	          j        |           |}|S )NT)stdoutstderruniversal_newlinesshell)
subprocessPopenPIPEcommunicatewarningswarn)cmderr_msgdefault_valueprocessr   r   s         h/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/launch/utils/topology.pycall_cmdr      se      G ((**NFF gM    c                   J    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 ZdS )SingleNodeTopologyc                     d| _         t          d          | _        d| _        d| _        i | _        g | _        g | _        d | _        d S )N        inf         )	pcie_latencyfloatpcie_bandwidthnvlink_bandwidth
nb_devicesmachinedeviceslinksjson_objectselfs    r   __init__zSingleNodeTopology.__init__(   sJ    #Ell $
r   c                    d}d}d}t          |||          }d}d}d}t          |||          }d}d}d	}t          |||          }d
}d}d}	d}d}d}t          |||          }
|
rd}d
}	d}d}d}t          |||          }
|
rd}d}	d}d}d}t          |||          }
|
rd}d}	t          |          t          |          z  t          |          z  |z  }||z  }||	z  }|| j        d<   || j        d<   d S )Nz,lscpu | grep 'Socket(s)' | awk '{print $NF}'zFailed to get number of sockets   z5lscpu | grep 'Core(s) per socket' | awk '{print $NF}'z(Failed to get number of cores per socket   zElscpu | grep GHz | awk -F '@' '{print $NF}' | awk -F 'G' '{print $1}'zFailed to get cpu clock rateg333333@   r   zlscpu | grep ssezFailed to get cpu vector sizessezlscpu | grep avx2avx2r   zlscpu | grep avx512avx512   	sp_gflops	dp_gflops)r   intr   r!   )r&   r   r   r   
nb_socketsnb_cores_per_socket
clock_ratenb_fmassimd_width_spsimd_width_dpvector_sizegflops_per_elementr0   r1   s                 r   calculate_cpu_flopsz&SingleNodeTopology.calculate_cpu_flops3   s   <3c7M::
 F<&sG]CC V0c7M::
   1sG];; 	MM!1sG];; 	MM#1 sG];; 	MM 
OO%&&'J   	 '6	&6	$-[!$-[!!!r   c                 f    |dk    rdS |dk    rdS |dk    rdS |dk    rdS |d	k    rd
S |dk    rdS d S )N   g      ?r+   g      ?   g      ?r)   g       @   g      @   g       @ )r&   pcie_generations     r   pcie_gen2bandwidthz%SingleNodeTopology.pcie_gen2bandwidthu   sj    a4!!3!!3!!3!!3!!3 "!r   c                 ^    d|v rd|v rdS d|v rd|v rdS d|v rdS d|v rdS d	|v rd
S d|v rdS d S )NH100SXM5)i`  i0u  PCIe)i  i]  A100)i,L  i%  A800V100)iT=  ix  P100)ih)  i  rA   )r&   models     r   model2gflopszSingleNodeTopology.model2gflops   sm    U??v<u__5<u__;u__;u__;u__; _r   c                 X   d|z   }d|z   }dt          |          z   dz   t          |          z   dz   }d}d}t          |||          }| j        }	d|v rU| j        dk    r)d	}d
}d}t	          t          |||                    | _        t          |dd                    | j        z  }	d}||	fS )Nr+   z cat /tmp/matrix.txt | awk 'FNR==z	 {print $z}'Failed to get topo matrixNVLNVr   z8nvidia-smi nvlink -s -i 0 | tail -n 1 | awk '{print $3}'zFailed to get nvlink bandwidth25)strr   r   r   r   r2   )
r&   	source_id	target_idrow_id	column_idr   r   r   	link_typelink_bandwidths
             r   get_link_bandwidthz%SingleNodeTopology.get_link_bandwidth   s    Y	M	 /&kk )nn 	 	 .S'=99	,9$,,P: $(-S'=99) )% !122//$2GGNI.((r   c                 \   d}d}d}t          |||                                          }d}d}d}t          |||                                          }d}d}d	}t          t          |||                    d
z  }|                                  || j        d<   || j        d<   || j        d<   d S )Nzhostname -szFailed to get hostname	localhostzhostname -izFailed to get host ip addressz	127.0.0.1zUcat /proc/meminfo | grep 'MemAvailable' | awk -F ':' '{print $NF}' | awk '{print $1}'zFailed to get cpu memory41366484g    .Ahostnameaddrmemory)r   stripr2   r;   r!   )r&   r   r   r   r^   ip_addr
cpu_memorys          r   get_host_infoz SingleNodeTopology.get_host_info   s    *#C-88>>@@ 1#377==?? f,"#w>>??3F
 	  """#+Z &V!+Xr   c           
         d}d}d}t          t          |||                    | _        t          t          j        d                    }|| j        k     r|| _        t          | j                  D ]}dt          |          z   dz   }d}d}t          t          |||                    }dt          |          z   d	z   }d
}d}t          t          |||                    }t          | j        | 	                    |          |z            | _        g }g }	g }
g }g }g }g }t          j
        j                                        }t          | j                  D ])}|                    ||z              |	                    |           |
                    d           dt          |          z   dz   }d}d}|                    t          |||                                                     dt          |          z   dz   }d}d}|                    t          t          |||                    dz             |                     ||                   \  }}|                    |           |                    |           +t          t!          |                    D ]k}i }||         |d<   |	|         |d<   |
|         |d<   ||         |d<   ||         |d<   ||         |d<   ||         |d<   | j                            |           l| j        | j        d<   | j        | j        d<   |
d         | j        d<   |
d          d|d          | j        d <   | j        | j        d!<   d S )"Nznvidia-smi -L | wc -lzFailed to get device count8PADDLE_LOCAL_SIZEznvidia-smi --id=z4 --query-gpu=pcie.link.gen.max --format=csv,noheaderz&Failed to get max pcie link generation4z6 --query-gpu=pcie.link.width.max --format=csv,noheaderz!Failed to get max pcie link width16GPUz' --query-gpu=name --format=csv,noheaderzFailed to get device namezNVIDIA A100-SXM4-40GBzA --query-gpu=memory.free --format=csv,noheader | awk '{print $1}'z%Failed to get device available memory40536g     @@	global_idlocal_idtyperL   r`   r0   r1   latency	bandwidthr   device_type-device_type_fullr"   )r2   r   r    osgetenvrangerS   minr   rC   paddledistributedget_rankappendra   rM   lenr"   r   r!   )r&   r   r   r   
local_sizeirB   
pcie_widthdev_global_idsdev_local_ids	dev_types
dev_modelsdev_memoriesdev_sp_gflopsdev_dp_gflops
rank_firstr0   r1   devices                      r   get_device_infoz"SingleNodeTopology.get_device_info   s   %.hsG]CCDD#67788
''(DOt'' 	 	A"a&&HI 
 ?GM!(3"G"GHHO #a&&JK 
 :G MXc7MBBCCJ"%#''88:E# #D
 	
 ].7799
t'' 	, 	,A!!!j.111  ###U### #a&&;< 
 2G3MhsG]CCIIKKLLL #a&&UV 
 >G#MHS'=99::cA   $(#4#4Z]#C#C Iy  +++  ++++s>**++ 		( 		(AF"0"3F;!.q!1F:&q\F6N(mF7O+AF8"/"2F;"/"2F;L''''"&"3Y$($7[!&/l]#.7l+L+LZ]+L+L'("&,Yr   c                    g }g }g }g }g }d}d}d}t          |||           t          j        j                                        }	t	          | j                  D ]%}
t	          | j                  D ]}|
|k    r+|                    d           |                    d           4|                    |
|	z              |                    ||	z              |                    d           |
|k    rD|| j        z  |
z   }|                    ||                    |                    ||                    |
|k     rC|                     |
|          \  }}|                    |           |                    |           't          t	          | j                            D ]B}
|	                    |
| j        z  |
z              |	                    |
| j        z  |
z              Cd}d}d}t	          t          |                    D ]U}
i }||
         |d	<   ||
         |d
<   ||
         |d<   ||
         |d<   ||
         |d<   | j                            |           V| j        | j        d<   d S )Nz$nvidia-smi topo -m > /tmp/matrix.txtrO    Xr   r   zrm /tmp/matrix.txtzFailed to delete matrix.txtsource_global_idtarget_global_idrn   ro   rp   r#   )r   rx   ry   rz   rv   r    r{   rZ   reversedpopr|   r#   r!   )r&   link_source_global_idslink_target_global_ids
link_typeslink_latencieslink_bandwidthsr   r   r   r   r~   jindexrX   rY   links                   r   get_link_infoz SingleNodeTopology.get_link_info&  s   !#!#
4-g}---].7799
t'' 	? 	?A4?++ ? ?66%%c***#**40000*11!j.AAA*11!j.AAA"))#...1uu !DO 3a 7"))*U*;<<<'..u/EFFFFQ484K4Kq5 51	> #)))444'..~>>>#?& %0011 	9 	9ANN1t.2333DO 3a 78888"/ s:'' 	$ 	$AD'=a'@D#$'=a'@D#$%a=DL,Q/DO / 2DJd#### $
Wr   c                     |                                   |                                  |                                  t          j        | j        d          | _        d S )Nr)   indent)rd   r   r   jsondumpsr!   r$   r%   s    r   detectzSingleNodeTopology.detect\  s[     	 	:dl1===r   c                     t          |d          5 }t          j        | j        |d           d d d            d S # 1 swxY w Y   d S )Nwr)   r   )openr   dumpr!   )r&   output_pathoutfiles      r   r   zSingleNodeTopology.dumph  s    +s## 	7wIdlGA6666	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7s   ;??N)__name__
__module____qualname__r'   r;   rC   rM   rZ   rd   r   r   r   r   rA   r   r   r   r   '   s        	  	  	 @. @. @.D    ) ) )@, , ,4Y/ Y/ Y/v4+ 4+ 4+l
> 
> 
>7 7 7 7 7r   r   )r   rt   r   r   rx   r   r   rA   r   r   <module>r      s     				         C7 C7 C7 C7 C7 C7 C7 C7 C7 C7r   