
    x-jr                       d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZmZ d dlZd dlmZ d dlmZ dd	lmZ e
rd dlmZ d d
lmZ ddgZdaej                            dej        j                                                  Z e ej                            dd                     Z!	 	 d"d#dZ"	 d$d%dZ# G d d          Z$ G d d          Z% G d d          Z& G d de&          Z' G d  d!          Z(dS )&    )annotationsN)reduce)product)TYPE_CHECKINGAnyLiteral)
NCCLConfig)check_nccl_version_for_p2p   )logger)GroupCommunicateTopologyHybridCommunicateGroupPADDLE_USE_FOUR_DIRECTIONS_P2P$FLAGS_pipeline_nccl_comm_init_optionmessage0NCCLConfig_Message | dict[str, int | str] | Nonedefault_name
str | Nonereturnr	   c                   t           j        j        j        dk    rd S t	          | t
          t          f          sd S ddlm} ddl	m
} t	          | t                    r| }n || d          }d|vr|||d<    |j        j        di |S )	Nncclr   )MessageToDict)coreT)preserving_proto_field_namecommName )paddledistributed
collective_default_backend
isinstanceNCCLConfig_Messagedictgoogle.protobuf.json_formatr   paddle.baser   r	   create)r   r   r   r   ret_dicts        f/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/fleet/base/topology.pymessage2nccl_configr*   /   s     $5??tg 2D9:: t999999      '4   L =dKKK!!l&>+!4?!--H---    nccl_configdict[str, int | str] | NoneNCCLConfig | Nonec                "    t          | d          S )a  

    Function that creates nccl config.

    Args:
        nccl_config (dict[str, int | str] | None): None or a dict containing the following keys:
            commName (str): name of the process group. ll_buffsize (int): buffer size of ll protocol.
            ll128_buffsize (int): buffer size of ll128 protocol. simple_buffsize (int): buffer size of
            simple protocol. buffsize_align (int): alignment unit of the total buffer size.
            nchannels (int): max number of channels. algoStr (str): communication algorithm.
            protoStr (str): communication protocol.

    Returns:
        NCCLConfig (NCCLConfig | None): an object containing the information,
        which can be used as an argument of new_group().

    Examples:
        .. code-block:: python

            >>> # doctest: +REQUIRES(env: DISTRIBUTED)
            >>> import paddle
            >>> import paddle.distributed as dist
            >>> from typing import Union
            >>> dist.init_parallel_env()
            >>> nccl_config: dict[str, Union[int, str]] = {"commName":"tp_comm","ll_buffsize":0,"ll128_buffsize":0,"simple_buffsize":1024,"buffsize_align":1024,"nchannels":4,"algoStr":"Ring","protoStr":"Simple",}
            >>> ranks=[0,1,2,3,4,5,6,7]
            >>> nccl_config=dist.create_nccl_config(nccl_config)
            >>> pg=dist.new_group(ranks, nccl_config=nccl_config)
            >>> m, n = 4096, 8192
            >>> local_rank = dist.get_rank(pg)
            >>> num_local_ranks = dist.get_world_size(pg)
            >>> x = paddle.ones(shape=[m, n], dtype=paddle.float32) * (local_rank + 1)
            >>> dist.all_reduce(x, group=pg)

    N)r*   r,   s    r)   create_nccl_configr1   E   s    L {D111r+   c                  &    e Zd ZdZdZdZdZdZdZdS )ParallelModea  

    There are all the parallel modes currently supported:

        - DATA_PARALLEL: Distribute input data to different devices.
        - TENSOR_PARALLEL: Shards tensors in the network to different devices.
        - PIPELINE_PARALLEL: Place different layers of the network on different devices.
        - SHARDING_PARALLEL: Segment the model parameters, parameter gradients and optimizer states corresponding to the parameters to each device.

    Examples:
        .. code-block:: python

            >>> # doctest: +REQUIRES(env: DISTRIBUTED)
            >>> import paddle
            >>> parallel_mode = paddle.distributed.ParallelMode
            >>> print(parallel_mode.DATA_PARALLEL)
            0

    r      r         N)	__name__
__module____qualname____doc__DATA_PARALLELTENSOR_PARALLELPIPELINE_PARALLELSHARDING_PARALLELSEGMENT_PARALLELr   r+   r)   r3   r3   n   s8         ( MOr+   r3   c                  t    e Zd Zg dg dfd d	Zd!d
Zd"dZd#dZd$dZd%dZd&dZ	d"dZ
d'dZd(dZd)dZdS )*r   )datapipeshardingsepcontextmodel)r4   r4   r4   r4   r4   r4   hybrid_group_names	list[str]dims	list[int]r   Nonec           
         | _         | _        t          j        d j                    _        t          d  j        d           _        d  j        D             } fdt          | D             }t          t          |t          t          |                                         _        t          t           j                                         j                                                             _        d S )N
Coordinatec                    | |z  S Nr   )xys     r)   <lambda>z.CommunicateTopology.__init__.<locals>.<lambda>   s
    q1u r+   r4   c                ,    g | ]}t          |          S r   )range).0ds     r)   
<listcomp>z0CommunicateTopology.__init__.<locals>.<listcomp>   s    ///q%((///r+   c                $    g | ]} j         | S r   )
coordinate)rU   rP   selfs     r)   rW   z0CommunicateTopology.__init__.<locals>.<listcomp>   s"    HHH!/$/1-HHHr+   )_parallel_names_dimscollections
namedtuplerY   r   _world_sizer   r$   ziprT   len_coord2rankvalueskeys_rank2coord)rZ   rG   rI   rangesall_coordinates   `    r)   __init__zCommunicateTopology.__init__   s      2
%0$.
 
 ""4"4dj!DD//DJ///HHHHw7GHHHNE#n:M:M4N4N O OPP ''))4+;+@+@+B+BCC
 
r+   c                    | j         S rO   )r[   rZ   s    r)   get_hybrid_group_namesz*CommunicateTopology.get_hybrid_group_names       ##r+   	axis_namestrintc                L    | j         | j                            |                   S rO   )r\   r[   indexrZ   rm   s     r)   get_dimzCommunicateTopology.get_dim   s     z$.44Y??@@r+   c                    | j         S rO   )r_   rj   s    r)   
world_sizezCommunicateTopology.world_size       r+   argsr   c                    t          |          t          | j                  k    sJ  | j        di |}|| j                                        v sJ | j        |         S Nr   )ra   r\   rY   rb   rd   )rZ   rw   keys      r)   get_rankzCommunicateTopology.get_rank   sf    4yyC
OO++++do%%%%d&++------$$r+   rankc                p    || j         k     sJ || j                                        v sJ | j        |         S rO   )r_   re   rd   )rZ   r|   s     r)   	get_coordzCommunicateTopology.get_coord   sD    d&&&&&t',,......%%r+   rq   c                      j                             |           fd j                                        D             }|                                 |S )Nc                B    g | ]}|         k    j         |         S r   )rb   )rU   coordaxisrq   rZ   s     r)   rW   z5CommunicateTopology.get_axis_list.<locals>.<listcomp>   s:     
 
 
T{e## U####r+   )r[   rq   rb   rd   sort)rZ   rm   rq   ranksr   s   ` ` @r)   get_axis_listz!CommunicateTopology.get_axis_list   sp    #)))44
 
 
 
 
 
)..00
 
 

 	

r+   c                b    || j         v sJ | j        | j                             |                   S rO   )r[   r\   rq   rr   s     r)   get_dim_sizez CommunicateTopology.get_dim_size   s4    D00000z$.44Y??@@r+   
fused_axislist[list[int]]c           
        t          t          | j                                      |                    }g }|D ]G}|                    t          | j        | j                            |                                        Hg }|D ]G}|                    t          | j        | j                            |                                        Hg }t          | D ]}i }g }	t          |          D ]\  }
}||||
         <   t          | D ]M}t          |          D ]\  }
}||||
         <   |	                    | j
         | j        di |                    N|                    |	           |S ry   )listsetr[   
differenceappendrT   r\   rq   r   	enumeraterb   rY   )rZ   r   non_fused_axisnon_fused_rangesrm   fused_ranges	rank_listnon_fused_ranks
coord_dictr   inon_fused_rankfused_ranks
fused_ranks                 r)   get_fused_ranksz#CommunicateTopology.get_fused_ranks   s   c$"677BB:NNOO' 	 	I##dj!5!;!;I!F!FGHH    # 	 	Idj!5!;!;I!F!FGHH    	&(89 		$ 		$OJE%.%?%? ? ?!>0>
>!,--&5 N N%.{%;%; ; ;MAz0:Jz!}--T-odo.K.K
.K.KLMMMMU####r+   c           
         | j         v sJ fd| j         D             }g }|D ]9}|                     |          }|                    t          |                     :g }t	          | D ]}i }|D ] }	||                    |	                   ||	<   !g }
t          d|                                         D ]2}||<   |
                    | j         | j        di |                    3|                    |
           |S )Nc                     g | ]
}|k    |S r   r   )rU   namerm   s     r)   rW   z5CommunicateTopology.get_comm_list.<locals>.<listcomp>   s*     
 
 
TY5F5FD5F5F5Fr+   r   r   )r[   r   r   rT   r   rq   rb   rY   )rZ   rm   other_axis_namesrf   r   dim_num
all_resultrP   	key_coord
other_nameresultr   s    `          r)   get_comm_listz!CommunicateTopology.get_comm_list   s[   D00000
 
 
 
!1
 
 
 $ 	* 	*D''--GMM%..))))
&! 		& 		&AI. N N
()*:*@*@*L*L(M	*%%F1d//	::;; N N'(	)$d.t/K/K/K/KLMMMMf%%%%r+   global_rankkwargsc                    |                      |          } |j        di |                                } | j        di |S ry   )r~   _replace_asdictr{   )rZ   r   r   r   tfs        r)   get_rank_from_stagez'CommunicateTopology.get_rank_from_stage   sN    {++U^%%f%%--//t}""r"""r+   N)rG   rH   rI   rJ   r   rK   )r   rH   )rm   rn   r   ro   r   ro   )rw   r   r   ro   )r|   ro   r   r   )rm   rn   rq   ro   r   rJ   )r   rJ   r   r   )rm   rn   r   r   )r   ro   r   r   r   ro   )r7   r8   r9   rh   rk   rs   ru   r{   r~   r   r   r   r   r   r   r+   r)   r   r      s       )
 )
 )
 -,,
 
 
 
 
6$ $ $ $A A A A       % % % %& & & &
   A A A A   6   2# # # # # #r+   c                     e Zd Z	 dGdHdZdId
ZdJdZdKdZ	 	 	 dLdMdZ	 	 	 dLdMdZdNdZ	dNdZ
dKdZdKdZdOdZdNdZdNdZdNdZdNd ZdPd"ZdNd#ZdQd$ZdNd%ZdNd&ZdPd'ZdNd(ZdNd)ZdNd*ZdNd+ZdNd,ZdNd-ZdNd.ZdPd/ZdNd0Z dPd1Z!dRd3Z"dNd4Z#dNd5Z$dNd6Z%dPd7Z&dNd8Z'dSdTd;Z(dUd?Z)dPd@Z*dPdAZ+dNdBZ,	 dGdVdFZ-dS )Wr   Ntopologyr   hybrid_configsNCCLConfig_Message | Noner   rK   c                   t           j                                        | _        t           j                                        | _        || _        | j                            d          | _        | j                            d          | _	        | j                            d          | _
        | j                            d          | _        | j                            d          | _        |                                 | _        |                                 | _        |                                 | _        |                                 | _        |                                 | _        |                                 s:J d| j         d| j	         d| j         d	| j
         d
| j         d| j                     |                     d|t3          |d         j        d          nd           \  | _        | _        t           j                            t          j        dgd          t           j        j        j         | j                   d}t          j!        |          |         rM| j        j        dk    r=| j        j"        #                    |t3          |d         j$        d          nd            |                     d|t3          |d         j%        d          nd           \  | _&        | _'        |                     d|t3          |d         j%        d          nd           \  | _(        | _)        |                     d|t3          |d         j%        d          nd           \  | _*        | _+        d | _,        | j        dk    rA|                     d|t3          |d         j%        d          nd           \  | _,        | _-        | .                    d|t3          |d         j/        d          nd           \  | _0        | _1        | j        dk    rA| .                    d|t3          |d         j/        d          nd           \  | _2        | _3        | j        dk    r| 4                    ddg|t3          |d         j%        d          nd           \  | _5        | _6        | 4                    ddg|t3          |d         j%        d           nd           \  | _7        | _8        | j        d!k    | _9        | j        | j
        dz
  k    | _:        | j
        dk    r`t           j;        j<        =                                rt}                       | ?                                 t          r| A                                 d"| j         d#| j	         d$| j         d%| j
         d&| j         d'| j         }|d(| j(         d)| j*         d*| j         d+| j&         d,| j,         d-| j0         z  }t          jC        |           | aDd S ).NrA   rF   rB   rC   rD   znranks: z
, mp_num: z, sharding_num: z
, pp_num: z
, dp_num: z, sep_num: 
pp_configspp_collr0   r4   int32dtypeopgroup$FLAGS_eager_communication_connectionpp_p2p
dp_configsdp
mp_configstpsharding_configssep_configsdp_checksharding_checkdp_sep_configsdp_seppp_tp_configspp_tpr   HybridParallelInfo: rank_id: , mp_degree: , sharding_degree: , pp_degree: , dp_degree: , sep_degree: , mp_group: ,  sharding_group: , pp_group: , dp_group: z, sep:group: , check/clip group: )Er   r   get_world_sizenranksr{   r   _topors   
_dp_degree
_mp_degree
_pp_degree_sharding_degree_sep_degree_get_data_parallel_id_data_parallel_id_get_model_parallel_id_model_parallel_id_get_sharding_parallel_id_sharding_parallel_id_get_sep_parallel_id_sep_parallel_id_get_pipe_parallel_idstage_id_check_valid_topo_set_comm_groupr*   coll_nccl_config	_pp_group_pp_comm_group
all_reducezerosReduceOpSUM	get_flagsprocess_groupeager_connect_ring_exchangep2p_nccl_configr,   	_dp_group_dp_comm_group	_mp_group_mp_comm_group_sharding_group_sharding_comm_group
_sep_group_sep_comm_group_set_check_groupcheck_nccl_config_check_group_check_comm_groupsharding_check_groupsharding_check_comm_groupcreate_fuse_group_dp_sep_group_dp_sep_comm_group_pp_mp_group_pp_mp_comm_groupis_first_stageis_last_stage	frameworkr   is_compiled_with_ncclr
   _set_p2p_prev_next_use_four_directions_set_four_directions_p2p_groupr   info_HYBRID_PARALLEL_GROUP)rZ   r   r   env_name	debug_strs        r)   rh   zHybridCommunicateGroup.__init__  s   
 (7799!-6688
*,,V44*,,W55*,,V44 $
 2 2: > >:--e44!%!;!;!=!="&"="="?"?%)%C%C%E%E" $ 9 9 ; ;2244%%'' 	
 	
 zt{  z  zdo  z  ztOd  z  zptp  z  z  LP  L[  z  z  hl  hx  z  z	
 	
'
 /3.B.B
 "- $"<0A9    /C 	/
 	/
++ 	%%L!G,,,!*.% 	& 	
 	
 	
 :H%%h/ 	")A--#1MM *5	 ,*<8H$  
 " N 	 	 	 /3.B.B
 "- $"<0<d    /C 	/
 	/
++ /3.B.B
 "- $"<0<d    /C 	/
 	/
++ ;?:N:N
 "- $"#56BJ    ;O 	;
 	;
7d7 a484H4H
 &1 (&}5A5    5I 	5 	51DOT1 594I4I
 "- $"<0BJ    5J 	5
 	5
141  1$$ %% &1	 (&'9:L(  
  & 
 
). a &&
 &1 (&'78Dh    ' 	 	"' 9=8N8N!
 &1 (&7CW    9O 	9 	95Dt5 #mq0!]t/BC ?Q$::<< -*,,,##%%%# 633555TD,< T T4? T T $ 5T TDHOT TbfbqT T BF  BRT T 	 	  [DN  [  [tG[  [  [imiw  [  [  FJ  FT  [  [  cg  cr  [  [  HL  HY  [  [  	[	I "&r+   Literal[0, 1, 2, 3, 4]c                   | j         dk    r8| j        dk    r-| j        dk    r"| j        dk    r| j        dk    rt
          j        S | j         dk    r-| j        dk    r"| j        dk    r| j        dk    rt
          j        S | j         dk    r"| j        dk    r| j        dk    rt
          j        S | j         dk    r| j        dk    rt
          j	        S | j         dk    rt
          j
        S d S Nr4   )r   r   r   r   r   r3   r;   r>   r?   r<   r=   rj   s    r)   get_parallel_modez(HybridCommunicateGroup.get_parallel_mode  s    Oq  1$$ A%%%**!##--Oq  1$$ A%%%))  11Oq  1$$ 1$$  00_!!do&9&9  //_q  11 ! r+   boolc                b    | j         | j        z  | j        z  | j        z  | j        z  | j        k    S rO   )r   r   r   r   r   r   rj   s    r)   r   z(HybridCommunicateGroup._check_valid_topo  sF    Ooo #$ 	
 {	
r+   c                0    | j         dk    s
J d            d S )Nr4   zsep not existr   rj   s    r)   _check_sep_existz'HybridCommunicateGroup._check_sep_exist  s$    !###_#####r+   rA   parallel_methodrn   topor,   r.   tuple[list[int], Group]c                V   g }d }|| j         }|                    |          }|dk    rt          nd}|D ]1}t          j                            |||          }	| j        |v r|}|	}2t          |          dk    sJ |J t          j	        dt          |           d| d           ||fS )NrB   r   r   nccl_comm_init_optionr,   Total  z# comm group(s) create successfully!)
r   r    g_pipeline_nccl_comm_init_optionr   r   	new_groupr   ra   r   r  )
rZ   r  r  r,   parallel_groupparallel_comm_groupparallel_groupsgroup_nccl_comm_init_optionr   
comm_groups
             r)   r   z&HybridCommunicateGroup._set_comm_group  s    "<:D,,_==  6)) -, 	$
 % 	1 	1E+55&A' 6  J
 5((!&&0#>""Q&&&&"...`S))``O```	
 	
 	
 222r+   c                6   g }d }|| j         }|                    |          }t          |          D ]K}| j                             ||          }t          j                            ||          }	| j        |v r|}|	}Lt          |          dk    sJ |J ||fS )Nr   r,   r   )	r   rs   rT   r   r   r   r"  r   ra   )
rZ   r  r  r,   r#  r$  parallel_sizeidxr%  r'  s
             r)   r   z'HybridCommunicateGroup._set_check_group  s     "<:D_55'' 	1 	1C"j66LLO+55%; 6  J ?22!0&0#>""Q&&&&"...222r+   ro   c                D    t          | d          s
J d            | j        S )N	next_rankznext_rank has not been inited)hasattrr-  rj   s    r)   _get_p2p_next_rankz)HybridCommunicateGroup._get_p2p_next_rank,  )    t[))JJ+JJJ)~r+   c                D    t          | d          s
J d            | j        S )N	prev_rankzprev_rank has not been inited)r.  r2  rj   s    r)   _get_p2p_prev_rankz)HybridCommunicateGroup._get_p2p_prev_rank0  r0  r+   c                $   | j                             d          }|D ]r}t          |          | j        k    sJ t	          |          D ]F\  }}|}||dz   | j        z           }||dz
  | j        z           }| j        |k    r|| _        || _        Gsd S )NrB   r4   )r   r   ra   r   r   r   r-  r2  )rZ   
comm_lists
comm_ranksr+  r|   	curr_rankr-  r2  s           r)   r	  z)HybridCommunicateGroup._set_p2p_prev_next4  s    Z--f55
$ 		/ 		/Jz??do5555&z22 / /	T 	&a4?'BC	&a4?'BC	#y00%.DN%.DN/		/ 		/r+   c                   | j                             d          }d | _        d | _        d | _        d | _        |D ]}t          |          | j        k    sJ t          |          D ]\  }}|}||dz   | j        z           }||dz
  | j        z           }t          j
                            ||g          }| j        |k    r|| _        n| j        |k    r|| _        t          j
                            ||g          }	| j        |k    r|	| _        | j        |k    r|	| _        | j        J | j        J | j        J | j        J d S )NrB   r4   )r   )r   r   send_next_groupsend_prev_grouprecv_next_grouprecv_prev_groupra   r   r   r   r   r"  r   )
rZ   r5  r6  r+  r|   r7  r-  r2  
next_group
prev_groups
             r)   r  z5HybridCommunicateGroup._set_four_directions_p2p_groupB  s   Z--f55
####$ 	6 	6Jz??do5555&z22 6 6	T 	&a4?'BC	&a4?'BC	#/99$i0 :  
 #y00+5D((%22+5D(#/99$i0 :  
 #y00+5D((%22+5D()6, #///#///#///#/////r+   c                    | j         S rO   )r   rj   s    r)   r   zHybridCommunicateGroup.topologyg  s
    zr+   c                    | j         S rO   )r   rj   s    r)   get_global_rankz&HybridCommunicateGroup.get_global_rankj  rv   r+   c                J    | j                             | j                  j        S rO   )r   r~   r   rA   rj   s    r)   r   z,HybridCommunicateGroup._get_data_parallel_idn      z##D$455::r+   c                    | j         S rO   )r   rj   s    r)   get_data_parallel_rankz-HybridCommunicateGroup.get_data_parallel_rankq  s    %%r+   c                    | j         S rO   )r   rj   s    r)   get_data_parallel_world_sizez3HybridCommunicateGroup.get_data_parallel_world_sizet  
    r+   r   c                    | j         S rO   )r   rj   s    r)   get_data_parallel_groupz.HybridCommunicateGroup.get_data_parallel_groupw      ""r+   c                &    | j         j        d         S Nr   )r   r   rj   s    r)    get_data_parallel_group_src_rankz7HybridCommunicateGroup.get_data_parallel_group_src_rankz      "(++r+   c                J    | j                             | j                  j        S rO   )r   r~   r   rF   rj   s    r)   r   z-HybridCommunicateGroup._get_model_parallel_id~  s    z##D$455;;r+   c                    | j         S rO   )r   rj   s    r)   get_model_parallel_rankz.HybridCommunicateGroup.get_model_parallel_rank  s    &&r+   c                    | j         S rO   )r   rj   s    r)   get_model_parallel_world_sizez4HybridCommunicateGroup.get_model_parallel_world_size  rH  r+   c                    | j         S rO   )r   rj   s    r)   get_model_parallel_groupz/HybridCommunicateGroup.get_model_parallel_group  rK  r+   c                &    | j         j        d         S rM  )r   r   rj   s    r)   !get_model_parallel_group_src_rankz8HybridCommunicateGroup.get_model_parallel_group_src_rank  rO  r+   c                J    | j                             | j                  j        S rO   )r   r~   r   rB   rj   s    r)   r   z,HybridCommunicateGroup._get_pipe_parallel_id  rC  r+   c                    | j         S rO   )r   rj   s    r)   get_stage_idz#HybridCommunicateGroup.get_stage_id  s
    }r+   c                    | j         S rO   )r   rj   s    r)   get_pipe_parallel_world_sizez3HybridCommunicateGroup.get_pipe_parallel_world_size  rH  r+   c                J    | j                             | j                  j        S rO   )r   r~   r   rD   rj   s    r)   r   z+HybridCommunicateGroup._get_sep_parallel_id  s    z##D$45599r+   c                    | j         S rO   )r   rj   s    r)   get_sep_parallel_rankz,HybridCommunicateGroup.get_sep_parallel_rank      $$r+   c                    | j         S rO   r  rj   s    r)   get_sep_parallel_world_sizez2HybridCommunicateGroup.get_sep_parallel_world_size  rv   r+   c                8    |                                   | j        S rO   )r  r   rj   s    r)   get_sep_parallel_groupz-HybridCommunicateGroup.get_sep_parallel_group  s    ##r+   c                N    |                                   | j        j        d         S rM  )r  r   r   rj   s    r)   get_sep_parallel_group_src_rankz6HybridCommunicateGroup.get_sep_parallel_group_src_rank  s%    #)!,,r+   c                    | j         S rO   )r   rj   s    r)   get_pipe_parallel_groupz.HybridCommunicateGroup.get_pipe_parallel_group  rK  r+   !tuple[Group, Group, Group, Group]c                X    t           s
J d            | j        | j        | j        | j        fS )NzrIf you want to use four directions p2p group, set the environment variable PADDLE_USE_FOUR_DIRECTIONS_P2P to True.)r
  r9  r:  r;  r<  rj   s    r)   get_p2p_groupsz%HybridCommunicateGroup.get_p2p_groups  sF    # 	
 	
 A	
 	
#     	
 	
r+   c                J    | j                             | j                  j        S rO   )r   r~   r   rC   rj   s    r)   r   z0HybridCommunicateGroup._get_sharding_parallel_id  s    z##D$455>>r+   c                    | j         S rO   )r   rj   s    r)   get_sharding_parallel_rankz1HybridCommunicateGroup.get_sharding_parallel_rank  s    ))r+   c                    | j         S rO   )r   rj   s    r)    get_sharding_parallel_world_sizez7HybridCommunicateGroup.get_sharding_parallel_world_size  ra  r+   c                    | j         S rO   )r   rj   s    r)   get_sharding_parallel_groupz2HybridCommunicateGroup.get_sharding_parallel_group      ((r+   c                &    | j         j        d         S rM  )r   r   rj   s    r)   $get_sharding_parallel_group_src_rankz;HybridCommunicateGroup.get_sharding_parallel_group_src_rank  s    (.q11r+   FrC   c                "    |r| j         S | j        S rO   )r   r   )rZ   rC   s     r)   get_check_parallel_groupz/HybridCommunicateGroup.get_check_parallel_group  s     	*11))r+   r   r   r   c                6     | j         j        | j        fd|i|S )NrB   )r   r   r   )rZ   r   r   s      r)   r   z*HybridCommunicateGroup.get_rank_from_stage  s6    -tz-
 
#+
/5
 
 	
r+   c                8    |                                   | j        S rO   )r  r  rj   s    r)   get_dp_sep_parallel_groupz0HybridCommunicateGroup.get_dp_sep_parallel_group  s    &&r+   c                8    |                                   | j        S rO   )r  r  rj   s    r)   get_pp_mp_parallel_groupz/HybridCommunicateGroup.get_pp_mp_parallel_group  s    %%r+   c                    dS rM  r   rj   s    r)   $get_moe_sharding_parallel_world_sizez;HybridCommunicateGroup.get_moe_sharding_parallel_world_size  s    qr+   fused_strategy_listrH   =tuple[list[list[int]], list[Group]] | tuple[list[int], Group]c                D   t          |          dk    s
J d            g }g }| j                            |          }|                                 |D ]V}t          j                            ||          }| j        |v r*|                    |           |                    |           Wt          |          dk    sJ t          |          dk    sJ t          j
        dt          |           d| d           t          |          dk    r||fS |d         |d         fS )Nr   z9the length of fused_strategy_list must be greater than 0.r)  r  z comm group(s) of fused z create successfully!r4   )ra   r   r   r   r   r   r"  r   r   r   r  )rZ   r  r,   r#  r$  r%  r   r'  s           r)   r   z(HybridCommunicateGroup.create_fuse_group  se   
 &''!+++G ,++  *445HII$ 	7 	7E+55 6  J 5((%%e,,,#**:666>""Q&&&&&''!++++mS))mmCVmmm	
 	
 	
 ~""!#666!!$&9!&<<<r+   rO   )r   r   r   r   r   rK   )r   r  r   r  r   rK   )rA   NN)r  rn   r  r   r,   r.   r   r  r   )r   r   r   r   )r   rn   )r   rj  F)rC   r  r   r   )r   ro   r   r   r   ro   )r  rH   r,   r.   r   r  ).r7   r8   r9   rh   r  r   r  r   r   r/  r3  r	  r  r   rA  r   rE  rG  rJ  rN  r   rR  rT  rV  rX  r   r[  r]  r   r`  rc  re  rg  ri  rl  r   ro  rq  rs  rv  rx  r   r{  r}  r  r   r   r+   r)   r   r     s        59x& x& x& x& x&t$2 $2 $2 $2L
 
 
 
5 5 5 5
  &$()-	!3 !3 !3 !3 !3J  &$()-	3 3 3 3 32      / / / /#0 #0 #0 #0J          ; ; ; ;& & & &   # # # #, , , ,< < < <' ' ' '   # # # #, , , ,; ; ; ;      : : : :% % % %       $ $ $ $- - - -# # # #	
 	
 	
 	
? ? ? ?* * * *% % % %) ) ) )2 2 2 2
* * * * *
 
 
 
' ' ' '& & & &    *.= = = = = = =r+   c                  (   e Zd Zg dg ddfd3dZd4dZd5dZd6dZd Zd Zd Z	d6dZ
d6dZ	 d6d7dZd Zd Zd Zd Zd8dZd Zd8dZd8d Zd9d"Zd8d#Zd9d$Zd8d%Zd9d&Zd8d'Zd8d(Zd8d)Zd9d*Zd8d+Zd8d,Zd8d-Z d9d.Z!d8d/Z"	 d:d8d1Z#d:d8d2Z$dS );EPHybridCommunicateGroup)rB   moe_shardingexpertrA   rC   rD   rE   rF   )r4   r4   r4   r4   r4   r4   r4   r4   NrG   rH   rI   rJ   r   r   r   rK   c                    t           j                                        | _        t           j                                        | _        t          t          ||                                        dd          | _	                            dd          | _
                            dd          | _                            dd          | _                            dd          | _                            dd          | _                            dd          | _                            dd          | _        d	vrdd	<                       d	d          | _        g }g }t          ||          D ]3\  }}|d
v r*|                    |           |                    |           4d|v rd|v sJ t'          ||          | _        d         d         z  d<   d |D             }|                    d          }	|                    d          |                    d          k    r0|                    |	dz   d           |                    |	d           n2|                    |	dz   d           |                    |	dz   d           fd|D             }
|                    d          |                    d          k     s
J d            t'          ||
          | _        d         d	         z  d<   g d}fd|D             }t'          ||          | _        | | j        _        | | j        _        | | j        _        | j        | _        |                     | j        d          | _        |                     | j        d          | _        |                                 | _        |                     | j        d          | _         |                     | j        d	          | _!        | j        "                    d          | _#        |                     | j        d          | _$        |                     | j        d          | _%        |                     | j        d          | _&        | j        | j        k    sJ d| j         d| j         d            | j        j'        | j        j'        k    s%J d| j        j'         d| j        j'         d            | j        dk    r| j        dk    sJ d| j         d| j         d            | (                    d| j        |tS          |d         j*        d          nd           \  | _+        | _,        t           j        -                    t          j.        dgd          t           j        j/        j0        | j,                    d!}t          j1        |          |         rM| j,        j        dk    r=| j,        j2        3                    |tS          |d         j4        d"          nd            | (                    d| j        |tS          |d#         j5        d$          nd           \  | _6        | _7        | (                    d| j        |tS          |d%         j5        d          nd           \  | _8        | _9        | (                    d| j        |tS          |d&         j5        d'          nd           \  | _:        | _;        | (                    d| j        |tS          |d(         j5        d          nd           \  | _<        | _=        | (                    d| j        |tS          |d)         j5        d*          nd           \  | _>        | _?        | @                    | j        |tS          |d+         j5        d          nd           \  | _A        | _B        | C                    | j        |tS          |d,         j5        d	          nd           \  | _D        | _E        d | _F        d | _G        | j        dk    rF| H                    | j        |tS          |d-         j5        d.          nd           \  | _F        | _G        | I                    | j        |tS          |d/         j5        d          nd           \  | _J        | _K        | L                                | _M        | N                    d| j        |tS          |d&         jO        d0          nd           \  | _P        | _Q        | N                    d| j        |tS          |d%         jO        d1          nd           \  | _R        | _S        | j$        d2k    | _T        | j$        | j        dz
  k    | _U        | j        dk    r`t           jV        jW        X                                rt                       | Z                                 t          r| \                                 d3| j         d4| j         d5| j         d6| j         d7| j         d8| j         d9| j         d:| j	         d;| j
         }|d<| j>         d=| jA         d>| j+         d?| j:         d@| j<         dA| jD         dB| jJ         dC| jF         dD| jP         dE| j6         dF| j8         dz  }t          j^        |           | a_d S )GNr  r4   r  rB   rA   rF   rC   rD   rE   )rB   r  r  dense_shardingc                    g | ]}|d v|	S ))r  rC   r  rE   r   )rU   r   s     r)   rW   z5EPHybridCommunicateGroup.__init__.<locals>.<listcomp>1  s.     
 
 
LLL LLLr+   r   c                     g | ]
}|         S r   r   rU   r   dim_dicts     r)   rW   z5EPHybridCommunicateGroup.__init__.<locals>.<listcomp>@  s    CCChtnCCCr+   z%moe_sharding must be before sharding.cp_sharding)r  rB   rE   rF   c                     g | ]
}|         S r   r   r  s     r)   rW   z5EPHybridCommunicateGroup.__init__.<locals>.<listcomp>P  s    ===d8D>===r+   zMismatch moe_pp_degree:z, pp_degree:.zMismatch world_size:z, moe_world_size:zsep_degree z and dp_degree z must be 1 in MoE.r   r   r0   r   r   r   r   r   
ep_configsepmoe_sharding_configsr   r   r   r   r   r   
cp_configscp_mp_configscp_mpcp_sharding_configs
data_checkmoe_sharding_checkr   r   r   r   r   r   r   z, cp_degree: z, ep_degree: z, moe_sharding_degree: r   r   r   r   z, sep_group: z, cp_group: z, cp_sharding_group: z, cp_mp_group: r   z, ep_group: z, moe_sharding_group: )`r   r   r   r   r{   r   r$   r`   get
_ep_degree_moe_sharding_degree_moe_pp_degreer   r   r   r   r   
_cp_degreer   r   	_moe_toporq   insert_dense_topo_cp_topo_parent_hcgr   _get_parallel_idr   r   r   r   r   _cp_parallel_idrs   _cp_sharding_degreer   _expert_parallel_id_moe_sharding_parallel_idr_   r   r*   r   r   r   r   r   r   r   r   r   r   r   r,   	_ep_group_ep_comm_group_moe_sharding_group_moe_sharding_comm_groupr   r   r   r   r   r   build_sharding_groupr   r   build_context_group	_cp_group_cp_comm_group_cp_mp_group_cp_mp_comm_groupbuild_cp_mp_fuse_groupbuild_context_sharding_group_cp_sharding_group_cp_sharding_comm_group_get_cp_sharding_parallel_id_cp_sharding_parallel_idr   r   r   r   r   r   r  r  r  r   r  r
   r	  r
  r  r   r  r  )rZ   rG   rI   r   moe_hybrid_group_namesmoe_dimsr   dimdense_group_namespipe_idx
dense_dimscp_group_namescp_dimsr  r  r  s                  @r)   rh   z!EPHybridCommunicateGroup.__init__  s9    (7799!-6688.5566",,x33$,LL$C$C!&ll6155",,vq11",,w22",,vq11 (Z ; ;#<<q11H$$"#HY",,y!44!#/66 	% 	%ID#999&--d333$$$44422223 --CXNN Z H^$<< 	!"
 
*
 
 

 %**622##F++.@.F.F/
 /
 
 
 $$X\3CDDD$$X~>>>>$$X\>BBB$$X\3CDDDCCCC1BCCC
 &&
 
##$4556 6 6 46 6 6 //@*MM"*:"6(9:M"M
 
 
 >===n===+NGDD%)"'+$$(!%
!%!6!6t7G!P!P"&"7"7g#
 #
 &*%C%C%E%E" $ 5 5d6F N N#44T]INN#'=#8#8#G#G --dnfEE#'#8#8NH$
 $
  *.)>)>NN*
 *
& "do555Yd&9YYtYYY 655 z%)CCCCi4:#9iiDNLfiii DCC 1$$A)=)=)=^$*^^4?^^^ *>)== /3.B.BN
 "- $"<0A9    /C 
/
 
/
++ 	%%L!G,,,!*.% 	& 	
 	
 	

 :H%%h/ 	")A--#1MM *5	 ,*<8H$  
 " N 	 	 	 /3.B.BN
 "- $"<0<d    /C 
/
 
/
++    &1	 (&'=>J&  
  !   	@ $"?  /3.B.B
 "- $"<0<d    /C 
/
 
/
++ 150D0D
 "- $"=1=u    1E 
1
 
1
-- /3.B.B
 "- $"<0<d    /C 
/
 
/
++ %%  &1	 (&'9:F"  
  & 
 
 	8d7 /3.F.F
 "- $"<0<i    /G 	/
 	/
++ !!%?Q++$
 *5 ,*?;G   " , 	 	 6Dt5 --  &1	 (&'<=I%  
  . 
 
 	>!= )-(I(I(K(K% 594I4I
 "- $"<0BL    5J 
5
 
5
141 !! &1 (&2+,	    "   	B!4#A$ #mq0!]t/BC ?Q$::<< -*,,,##%%%# 633555^D,< ^ ^4? ^ ^ $ 5^ ^DHO^ ^bfbq^ ^ BF  BR^ ^/^ ^ /^ ^ CGB[^ ^ 	 	  TDN  T  TtG[  T  Timiw  T  T  FJ  FT  T  T  cg  cr  T  T  @D  @N  T  T  ei  e|  T  T  MQ  M^  T  T  tx  tE  T  T  SW  Sa  T  T  y}  yQ  T  T  T  	T	I "&r+   r  c                    | j         | j        z  | j        z  | j        z  | j        z  | j        k    o| j        dk    p
| j        dk    S r  )r   r   r   r   r   r   r  rj   s    r)   r   z*EPHybridCommunicateGroup._check_valid_topoG  sd    Ooo #$ 	
 {> !#<t'71'<	>r+   c                0    | j         dk    s
J d            d S )Nr4   zcp not existr  rj   s    r)   _check_cp_existz(EPHybridCommunicateGroup._check_cp_existQ  s#    """N"""""r+   c                (   g }d }|                      |dd          }d}|D ]1}t          j                            |||          }| j        |v r|}|}2t          |          dk    sJ |J t          j        dt          |           d           ||fS )Nr  r  r   r  r  z, sharding comm group(s) create successfully!)merge_inner_comm_listr   r   r"  r   ra   r   r  )	rZ   r  r,   r#  r$  r%  r&  r   r'  s	            r)   r  z-EPHybridCommunicateGroup.build_sharding_groupT  s    "44."2
 
 '(#$ 	1 	1E+55&A' 6  J
 5((!&&0#>""Q&&&&"...WS))WWW	
 	
 	
 222r+   c                X   |                      |dd          }g }|D ]}t          |          | j        z  | j        k    s"J dt          |           d| j                     t	          | j                  D ]4}||| j        z  |dz   | j        z           }|                    |           5|S )Nr  r  sharding comm list ) size must divided by cp_sharding_degree r4   r  ra   r  r  rT   r   rZ   r  sharding_comm_listcontext_comm_listr   r   	sub_rankss          r)   split_context_comm_listz0EPHybridCommunicateGroup.split_context_comm_listp  s    !77."2
 
 ' 	4 	4Eu::!99T_LLLuc%jjuu[_[suu MLL 4344 4 4!'1q5DO*CC	 "((3333	4
 ! r+   c                >   |                      |dd          }g }|D ]}t          |          | j        z  | j        k    s"J dt          |           d| j                     t	          | j                  D ]'}||d | j                 }|                    |           (|S )Nr  r  r  r  r  r  s          r)    split_context_sharding_comm_listz9EPHybridCommunicateGroup.split_context_sharding_comm_list  s    !77."2
 
 ' 	4 	4Eu::!99T_LLLuc%jjuu[_[suu MLL 4?++ 4 4!!"6t"67	!((33334 ! r+   c                   |                     d          }|                     |          } G d d          } |            }||z   D ]Y}t          |          dk    rD|d         }t          dt          |                    D ]}|                    |||                    Z|                                }	|	D ]}
|
                                 |	                    d            |	S )NrF   c                  &    e Zd Zd Zd Zd Zd ZdS )REPHybridCommunicateGroup.fuse_context_tensor_parallel_comm_list.<locals>.UnionFindc                "    i | _         i | _        d S rO   )parentr|   rj   s    r)   rh   z[EPHybridCommunicateGroup.fuse_context_tensor_parallel_comm_list.<locals>.UnionFind.__init__  s     			r+   c                    || j         vr|| j         |<   d| j        |<   |S | j         |         |k    r(|                     | j         |                   | j         |<   | j         |         S rM  )r  r|   find)rZ   rP   s     r)   r  zWEPHybridCommunicateGroup.fuse_context_tensor_parallel_comm_list.<locals>.UnionFind.find  sd    DK''%&DKN#$DIaLH;q>Q&&%)YYt{1~%>%>DKN{1~%r+   c                $   |                      |          |                      |          }}||k    rd S | j        |         | j        |         k     r||}}|| j        |<   | j        |         | j        |         k    r| j        |xx         dz  cc<   d S d S r  )r  r|   r  )rZ   rP   rQ   pxpys        r)   unionzXEPHybridCommunicateGroup.fuse_context_tensor_parallel_comm_list.<locals>.UnionFind.union  s    1tyy||B88F9R=49R=00B"$B9R=DIbM11IbMMMQ&MMMMM 21r+   c                    i }| j         D ];}|                     |          }||vrg ||<   ||                             |           <t          |                                          S rO   )r  r  r   r   rc   )rZ   
componentsnoderoots       r)   get_componentszaEPHybridCommunicateGroup.fuse_context_tensor_parallel_comm_list.<locals>.UnionFind.get_components  sr    
 K 2 2D99T??D:--+-
4(t$++D1111J--//000r+   N)r7   r8   r9   rh   r  r  r  r   r+   r)   	UnionFindr    sP          & & &
' 
' 
'1 1 1 1 1r+   r  r4   r   c                    | d         S rM  r   )rP   s    r)   rR   zQEPHybridCommunicateGroup.fuse_context_tensor_parallel_comm_list.<locals>.<lambda>  s
    1Q4 r+   )rz   )r   r  ra   rT   r  r  r   )rZ   r  mp_comm_listcp_comm_listr  ufr   firstr   cp_tp_comm_list	components              r)   &fuse_context_tensor_parallel_comm_listz?EPHybridCommunicateGroup.fuse_context_tensor_parallel_comm_list  s   ))'2233D99"	1 "	1 "	1 "	1 "	1 "	1 "	1 "	1H Y[[!L0 	. 	.E5zzA~~aq#e**-- . .AHHUE!H----++--( 	 	INN000r+   c                   d}|                      |          }|D ]1}t          j                            |||          }| j        |v r|}|}2t          |          dk    sJ |J t          j        d| j         d           ||fS )Nr   r  r  z4 context parallel comm group(s) create successfully!)	r  r   r   r"  r   ra   r   r  r  	rZ   r  r,   r&  r%  r   r'  r#  r$  s	            r)   r  z,EPHybridCommunicateGroup.build_context_group  s    &'#66t<<$ 	1 	1E+55&A' 6  J
 5((!&&0#>""Q&&&&"...ZT_ZZZ	
 	
 	
 222r+   c                   d}|                      |          }|D ]1}t          j                            |||          }| j        |v r|}|}2t          |          dk    sJ |J t          j        d| j         d           ||fS )Nr   r  r  z= context sharding parallel comm group(s) create successfully!)	r  r   r   r"  r   ra   r   r  r  r  s	            r)   r  z5EPHybridCommunicateGroup.build_context_sharding_group  s    &'#??EE$ 	1 	1E+55&A' 6  J
 5((!&&0#>""Q&&&&"...lT-lll	
 	
 	
 222r+   r  c                    d}|                      |          }|D ]1}t          j                            |||          }| j        |v r|}|}2t          j        d           ||fS )Nr   r  z9Fused context & model parallel group create successfully!)r  r   r   r"  r   r   r  r  s	            r)   r  z/EPHybridCommunicateGroup.build_cp_mp_fuse_group  s     '(#EEdKK$ 	1 	1E+55&A' 6  J
 5((!&&0#OPPP222r+   c                |   |j                             |          }|j                             |          }|                    |          }t          |          |j        |         z  }t          j        |j        |dz   d                   |j        |         z  }|dk    r|dk    sJ g }	t          |          D ]}
g }t          |j        |                   D ]O}|
||z  z   t          |          k     s#J d|
 d| d| dt          |                       |||
||z  z            z  }P|	                    |           |	S )a$  
        merge all inner communication list whose rank-id are in
        the same outer communication list. E.g.:
          outer_comm_list: [[0, 4], [1, 5]]
          inner_comm_list: [[0, 2], [1, 3], [4, 6], [5, 7]]
          => merged_inner_comm_list: [[0, 2, 4, 6], [1, 3, 5, 7]]
        r4   Nr   z+Unexpected error in merge_inner_comm_list, z, )	r[   rq   r   ra   r\   mathprodrT   r   )rZ   r  
outer_name
inner_name
inner_axis
outer_axisinner_comm_listnum_merged_groupsintervalmerged_comm_listr   commjs                r)   r  z.EPHybridCommunicateGroup.merge_inner_comm_list  sw    )//
;;
)//
;;
,,Z8800DJz4JJIdj*q.!3!3455J9OO 	 !1$$A5()) 	* 	*AD4:j122 : :1x<'#o*>*>>>>n!nnqnnHnnX[\kXlXlnn ?>> AL(899##D))))r+   c                    t          |          }t          |d                   }|dz
  }d}|dk    r9||k     r3||         |         }||k    r|S ||k     r|dz  }n|dz  }|dk    r||k     3d S )Nr   r4   )ra   )rZ   	comm_listr   rowscolsrccurrents           r)   find_col_idxz%EPHybridCommunicateGroup.find_col_idx  s    9~~9Q<  1H1ffTl1oG+%%;&&QQ 1ffT tr+   c                n    |                     |          }|                     || j                  }|J |S rO   )r   r  r   )rZ   r  parallel_typer   parallel_ids        r)   r  z)EPHybridCommunicateGroup._get_parallel_id0  s?    &&}55	''	43CDD&&&r+   c                |    |                      | j        dd          }|                     || j                  }|J |S )Nr  r  )r  r  r  r   )rZ   r  r	  s      r)   r   z2EPHybridCommunicateGroup._get_sharding_parallel_id6  sP    !77n.>
 
 ''(:D<LMM&&&r+   ro   c                @    | j                             | j                  S rO   )r  rq   r   rj   s    r)   _get_context_parallel_idz1EPHybridCommunicateGroup._get_context_parallel_id>  s    ~##D$4555r+   c                @    | j                             | j                  S rO   )r  rq   r   rj   s    r)   r  z5EPHybridCommunicateGroup._get_cp_sharding_parallel_idA  s    &,,T-=>>>r+   c                    | j         S rO   )r  rj   s    r)   get_context_parallel_rankz2EPHybridCommunicateGroup.get_context_parallel_rankD  rl   r+   c                    | j         S rO   r  rj   s    r)   get_context_parallel_world_sizez8EPHybridCommunicateGroup.get_context_parallel_world_sizeG  rH  r+   r   c                8    |                                   | j        S rO   )r  r  rj   s    r)   get_context_parallel_groupz3EPHybridCommunicateGroup.get_context_parallel_groupJ  s    ""r+   c                N    |                                   | j        j        d         S rM  )r  r  r   rj   s    r)   #get_context_parallel_group_src_rankz<EPHybridCommunicateGroup.get_context_parallel_group_src_rankN  s%    "(++r+   c                8    |                                   | j        S rO   )r  r  rj   s    r)   get_cp_sharding_parallel_groupz7EPHybridCommunicateGroup.get_cp_sharding_parallel_groupR  s    ++r+   c                N    |                                   | j        j        d         S rM  )r  r  r   rj   s    r)   'get_cp_sharding_parallel_group_src_rankz@EPHybridCommunicateGroup.get_cp_sharding_parallel_group_src_rankV  s%    +1!44r+   c                8    |                                   | j        S rO   )r  r  rj   s    r)   get_cp_mp_parallel_groupz1EPHybridCommunicateGroup.get_cp_mp_parallel_groupZ  s    %%r+   c                N    |                                   | j        j        d         S rM  )r  r  r   rj   s    r)   !get_cp_mp_parallel_group_src_rankz:EPHybridCommunicateGroup.get_cp_mp_parallel_group_src_rank^  s%    %+A..r+   c                    | j         S rO   )r  rj   s    r)   get_expert_parallel_rankz1EPHybridCommunicateGroup.get_expert_parallel_rankb  s    ''r+   c                    | j         S rO   )r  rj   s    r)   get_expert_parallel_world_sizez7EPHybridCommunicateGroup.get_expert_parallel_world_sizee  rH  r+   c                    | j         S rO   )r  rj   s    r)   get_expert_parallel_groupz2EPHybridCommunicateGroup.get_expert_parallel_grouph  rK  r+   c                &    | j         j        d         S rM  )r  r   rj   s    r)   "get_expert_parallel_group_src_rankz;EPHybridCommunicateGroup.get_expert_parallel_group_src_rankk  rO  r+   c                    | j         S rO   )r  rj   s    r)   get_moe_sharding_parallel_rankz7EPHybridCommunicateGroup.get_moe_sharding_parallel_rankn  s    --r+   c                    | j         S rO   )r  rj   s    r)   r  z=EPHybridCommunicateGroup.get_moe_sharding_parallel_world_sizeq  rt  r+   c                    | j         S rO   )r  rj   s    r)   get_moe_sharding_parallel_groupz8EPHybridCommunicateGroup.get_moe_sharding_parallel_groupt  s    ,,r+   c                &    | j         j        d         S rM  )r  r   rj   s    r)   (get_moe_sharding_parallel_group_src_rankzAEPHybridCommunicateGroup.get_moe_sharding_parallel_group_src_rankw  s    ,2155r+   Fc                "    |r| j         S | j        S rO   )r  r   rZ   with_context_parallels     r)   rq  z9EPHybridCommunicateGroup.get_sharding_parallel_world_sizez  s     ! 	)++((r+   c                "    |r| j         S | j        S rO   )r  r   r.  s     r)   ro  z3EPHybridCommunicateGroup.get_sharding_parallel_rank  s      	.00--r+   )rG   rH   rI   rJ   r   r   r   rK   r  r  rO   )r   r  r   r  r  )%r7   r8   r9   rh   r   r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r!  r#  r%  r'  r  r*  r,  rq  ro  r   r+   r)   r  r    s       	)
 	)
 	)
 32248C& C& C& C& C&J
> > > >3 3 3 33 3 3 38! ! ! ! ! !5 5 5n3 3 3 3*3 3 3 3, !%3 3 3 3 3$     <  "    6 6 6 6? ? ?$ $ $ $   # # # #, , , ,, , , ,5 5 5 5& & & &/ / / /( ( ( (   # # # #, , , ,. . . .) ) ) )- - - -6 6 6 6 %*) ) ) ) ). . . . . . .r+   r  c                  6    e Zd ZdZd Zd Zd Zd Zd Zd Z	dS )	_CommunicateGroupztmp for staticc                    | a i | _        d S rO   )r  groupsrj   s    r)   rh   z_CommunicateGroup.__init__  s    !%r+   c                f    t           j        j                            |||          }|| j        |<   d S rO   )r   r   r    r   r4  )rZ   
group_name
group_rank
group_sizering_idgroup_ranksr   s          r)   set_comm_groupz _CommunicateGroup.set_comm_group  s:     "-33
 
 #(Jr+   c                2    || j         v sJ | j         |         S rO   )r4  )rZ   r6  s     r)   	get_groupz_CommunicateGroup.get_group  s"    T[(((({:&&r+   c                ,    |                      d          S NrF   )r=  rj   s    r)   rV  z*_CommunicateGroup.get_model_parallel_group  s    ~~g&&&r+   c                6    |                      d          j        S r?  )r=  r   rj   s    r)   rT  z/_CommunicateGroup.get_model_parallel_world_size  s    ~~g&&--r+   c                6    |                      d          j        S r?  )r=  r|   rj   s    r)   rR  z)_CommunicateGroup.get_model_parallel_rank  s    ~~g&&++r+   N)
r7   r8   r9   r:   rh   r;  r=  rV  rT  rR  r   r+   r)   r2  r2    st          
( ( (' ' '' ' '. . ., , , , ,r+   r2  )NN)r   r   r   r   r   r	   rO   )r,   r-   r   r.   ))
__future__r   r]   r  os	functoolsr   	itertoolsr   typingr   r   r   r   7paddle.distributed.fleet.proto.distributed_strategy_pb2r	   r#   #paddle.distributed.utils.nccl_utilsr
   utils.log_utilr   paddle.base.libpaddlepaddle.distributed.collectiver   __all__r  environr  baser   is_compiled_with_xpur
  ro   r!  r*   r1   r3   r   r   r  r2  r   r+   r)   <module>rP     s   # " " " " "      				             . . . . . . . . . .       K J J J J J # # # # # # 4000000333333 ":
; z~~$fk&6&K&K&M&M   $'3JNN91==$ $   AE#. . . . .. 04&2 &2 &2 &2 &2R       8u# u# u# u# u# u# u# u#p|= |= |= |= |= |= |= |=~E
. E
. E
. E
. E
.5 E
. E
. E
.P, , , , , , , , , ,r+   