
    Αi/                     F    S SK r S SKrS SKrS SKrS SKrS r " S S5      rg)    Nc                     [         R                  " U [         R                  [         R                  SSS9nUR                  5       u  pEU(       a  [        R
                  " U5        UnU$ )NT)stdoutstderruniversal_newlinesshell)
subprocessPopenPIPEcommunicatewarningswarn)cmderr_msgdefault_valueprocessr   r   s         h/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/launch/utils/topology.pycall_cmdr      sT    G ((*NFgM    c                   P    \ rS rSrS rS rS rS rS rS r	S r
S	 rS
 rS rSrg)SingleNodeTopology'   c                     SU l         [        S5      U l        SU l        SU l        0 U l        / U l        / U l        S U l        g )N        inf         )	pcie_latencyfloatpcie_bandwidthnvlink_bandwidth
nb_devicesmachinedeviceslinksjson_objectselfs    r   __init__SingleNodeTopology.__init__(   sB    #El $
r   c                    SnSnSn[        XU5      nSnSnSn[        XU5      nSnSnS	n[        XU5      nS
nSnSn	SnSnSn[        XU5      n
U
(       a  SnS
n	SnSnSn[        XU5      n
U
(       a  SnSn	SnSnSn[        XU5      n
U
(       a  SnSn	[        U5      [        U5      -  [        U5      -  U-  nX-  nX-  nXR                  S'   XR                  S'   g )Nz,lscpu | grep 'Socket(s)' | awk '{print $NF}'zFailed to get number of sockets   z5lscpu | grep 'Core(s) per socket' | awk '{print $NF}'z(Failed to get number of cores per socket   zElscpu | grep GHz | awk -F '@' '{print $NF}' | awk -F 'G' '{print $1}'zFailed to get cpu clock rateg333333@   r   zlscpu | grep ssezFailed to get cpu vector sizessezlscpu | grep avx2avx2r   zlscpu | grep avx512avx512   	sp_gflops	dp_gflops)r   intr   r"   )r'   r   r   r   
nb_socketsnb_cores_per_socket
clock_ratenb_fmassimd_width_spsimd_width_dpvector_sizegflops_per_elementr2   r3   s                 r   calculate_cpu_flops&SingleNodeTopology.calculate_cpu_flops3   sB   <3cM:
 F<&s]C V0cM:
   1s];MM!1s];MM#1 s];MM 
O%&'J   	 '6	&6	$-[!$-[!r   c                 X    US:X  a  gUS:X  a  gUS:X  a  gUS:X  a  gUS	:X  a  g
US:X  a  gg )N   g      ?r-   g      ?   g      ?r+   g       @   g      @   g       @ )r'   pcie_generations     r   pcie_gen2bandwidth%SingleNodeTopology.pcie_gen2bandwidthu   sI    a!!!!! "r   c                 p    SU;   a  SU;   a  gSU;   a  SU;   a  gSU;   a  gSU;   a  gS	U;   a  g
SU;   a  gg )NH100SXM5)i`  i0u  PCIe)i  i]  A100)i,L  i%  A800V100)iT=  ix  P100)ih)  i  rD   )r'   models     r   model2gflopsSingleNodeTopology.model2gflops   sO    U?vu_5u_u_u_u_ r   c                 :   SU-   nSU-   nS[        U5      -   S-   [        U5      -   S-   nSnSn[        XVU5      nU R                  n	SU;   aM  U R                  S:X  a   S	nS
nSn[	        [        XVU5      5      U l        [        USS  5      U R                  -  n	SnX4$ )Nr-   z cat /tmp/matrix.txt | awk 'FNR==z	 {print $z}'Failed to get topo matrixNVLNVr   z8nvidia-smi nvlink -s -i 0 | tail -n 1 | awk '{print $3}'zFailed to get nvlink bandwidth25)strr   r   r    r   r4   )
r'   	source_id	target_idrow_id	column_idr   r   r   	link_typelink_bandwidths
             r   get_link_bandwidth%SingleNodeTopology.get_link_bandwidth   s    Y	M	 /&k )n 	 	 .S=9	,,9$$,P: $(-S=9)% !12/$2G2GGNI((r   c                 4   SnSnSn[        XU5      R                  5       nSnSnSn[        XU5      R                  5       nSnSnS	n[        [        XU5      5      S
-  nU R                  5         X@R                  S'   XPR                  S'   X`R                  S'   g )Nzhostname -szFailed to get hostname	localhostzhostname -izFailed to get host ip addressz	127.0.0.1zUcat /proc/meminfo | grep 'MemAvailable' | awk -F ':' '{print $NF}' | awk '{print $1}'zFailed to get cpu memory41366484g    .Ahostnameaddrmemory)r   stripr4   r=   r"   )r'   r   r   r   rd   ip_addr
cpu_memorys          r   get_host_info SingleNodeTopology.get_host_info   s    *#C-8>>@ 1#37==? f,"#>?3F
 	  "#+Z &V!+Xr   c           
         SnSnSn[        [        XU5      5      U l        [        [        R                  " S5      5      nX@R                  :  a  X@l        [        U R                  5       H  nS[        U5      -   S-   nSnSn[        [        XU5      5      nS[        U5      -   S	-   nS
nSn[        [        XU5      5      n[        U R                  U R                  U5      U-  5      U l        M     / n/ n	/ n
/ n/ n/ n/ n[        R                  R                  R                  5       n[        U R                  5       H  nUR                  X_-   5        U	R                  U5        U
R                  S5        S[        U5      -   S-   nSnSnUR                  [        XU5      R                  5       5        S[        U5      -   S-   nSnSnUR                  [        [        XU5      5      S-  5        U R                  X   5      u  nnUR                  U5        UR                  U5        M     [        [!        U5      5       HQ  n0 nX   US'   X   US'   X   US'   X   US'   X   US'   X   US'   X   US'   U R"                  R                  U5        MS     U R$                  U R&                  S'   U R                  U R&                  S'   U
S   U R&                  S'   U
S    SUS    3U R&                  S '   U R"                  U R&                  S!'   g )"Nznvidia-smi -L | wc -lzFailed to get device count8PADDLE_LOCAL_SIZEznvidia-smi --id=z4 --query-gpu=pcie.link.gen.max --format=csv,noheaderz&Failed to get max pcie link generation4z6 --query-gpu=pcie.link.width.max --format=csv,noheaderz!Failed to get max pcie link width16GPUz' --query-gpu=name --format=csv,noheaderzFailed to get device namezNVIDIA A100-SXM4-40GBzA --query-gpu=memory.free --format=csv,noheader | awk '{print $1}'z%Failed to get device available memory40536g     @@	global_idlocal_idtyperP   rf   r2   r3   latency	bandwidthr   device_type-device_type_fullr#   )r4   r   r!   osgetenvrangerX   minr   rF   paddledistributedget_rankappendrg   rQ   lenr#   r   r"   )r'   r   r   r   
local_sizeirE   
pcie_widthdev_global_idsdev_local_ids	dev_types
dev_modelsdev_memoriesdev_sp_gflopsdev_dp_gflops
rank_firstr2   r3   devices                      r   get_device_info"SingleNodeTopology.get_device_info   s4   %.hs]CD#678
'(Ot'A"a&HI 
 ?GM!(3"GHO #a&JK 
 :G MXcMBCJ"%##''8:E#D' (0 	
 ]]..779
t'A!!!.1  #U# #a&;< 
 2G3Mhs]CIIKL #a&UV 
 >G#MHS=9:cA $(#4#4Z]#C Iy  +  +7 (: s>*+AF"0"3F;!.!1F:&\F6N(mF7O+F8"/"2F;"/"2F;LL' , #'"3"3Y$($7$7[!&/l]#.7l^1Z]O+L'("&,,Yr   c                 n   / n/ n/ n/ n/ nSnSnSn[        XgU5        [        R                  R                  R                  5       n	[	        U R
                  5       H  n
[	        U R
                  5       H  nX:X  a$  UR                  S5        UR                  S5        M,  UR                  X-   5        UR                  X-   5        UR                  S5        X:  a9  XR
                  -  U
-   nUR                  X<   5        UR                  X\   5        M  X:  d  M  U R                  X5      u  pUR                  U5        UR                  U5        M     M     [        [	        U R
                  5      5       HC  n
UR                  XR
                  -  U
-   5        UR                  XR
                  -  U
-   5        ME     SnSnSn[	        [        U5      5       HC  n
0 nX   US	'   X*   US
'   X:   US'   XJ   US'   XZ   US'   U R                  R                  U5        ME     U R                  U R                  S'   g )Nz$nvidia-smi topo -m > /tmp/matrix.txtrT    Xr   r   zrm /tmp/matrix.txtzFailed to delete matrix.txtsource_global_idtarget_global_idru   rv   rw   r$   )r   r   r   r   r}   r!   r   r_   reversedpopr   r$   r"   )r'   link_source_global_idslink_target_global_ids
link_typeslink_latencieslink_bandwidthsr   r   r   r   r   jindexr]   r^   links                   r   get_link_info SingleNodeTopology.get_link_info&  s   !#!#
4-}-]]..779
t'A4??+6%%c*#**40*11!.A*11!.A"))#.u !OO 3a 7"))**;<'../EF484K4K51	 #)))4'..~># , (( %01ANN1.23OO 3a 78 2 #/ s:'AD'='@D#$'='@D#$%=DL,/DO / 2DJJd# ( !%

Wr   c                     U R                  5         U R                  5         U R                  5         [        R                  " U R
                  SS9U l        g )Nr+   indent)rj   r   r   jsondumpsr"   r%   r&   s    r   detectSingleNodeTopology.detect\  sA     	 	::dll1=r   c                     [        US5       n[        R                  " U R                  USS9  S S S 5        g ! , (       d  f       g = f)Nwr+   r   )openr   dumpr"   )r'   output_pathoutfiles      r   r   SingleNodeTopology.dumph  s.    +s#wIIdllGA6 $##s	   !7
A)r#   r%   r$   r"   r!   r    r   r   N)__name__
__module____qualname____firstlineno__r(   r=   rF   rQ   r_   rj   r   r   r   r   __static_attributes__rD   r   r   r   r   '   s:    	 @.D)@,4Y/v4+l
>7r   r   )r   r{   r   r   r   r   r   rD   r   r   <module>r      s'     	    C7 C7r   