
    Αi*5                       % S SK Jr  S SKrS SKrS SKJrJr  S SKJr  S SK	r	S SK
Jr  S SKJr  SSKJrJrJr  SS	KJrJrJrJrJrJrJrJrJrJrJrJrJr  \(       a  \S
   r S\!S'   S SK"J#r#  / r$Sq%S r&0 q'S r(0 q)0 q*Sq+/ SQr,Sq-Sq.\R^                  " SS9r0S q1S r2S r3S r4S r5S r6S r7S r8S r9S r:S r;   S(S jr<Sq=S r>SS\0S S4           S)S jjr?S*S  jr@S+S! jrA0 qBS" rCS# rDS$ rES% rFS,S-S& jjrGS,S-S' jjrHg).    )annotationsN)TYPE_CHECKINGLiteral)	TypeAlias)core)in_dynamic_mode   )Group_add_new_groupis_initialized)	_c_concat_c_identity_c_lookup_table_c_softmax_with_cross_entropy)_c_softmax_with_multi_label_cross_entropy_c_split_Linear_linear_mp_allreduce_parallel_embedding_parallel_linear_set_var_distributedsplit)glooncclxcclbkclflagcxr   _BackendList)
NCCLConfigc                 `    [         (       d  [        R                  R                  5       q [         $ N)_global_envpaddledistributedParallelEnv     ]/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/collective.py_get_global_envr*   :   s!    ;((446r(   _default_pg)r   r   heterr   r   r   i  )secondsc                    U q g r"   )_default_backend)backends    r)   _set_default_backendr1   X   s    r(   c                    U q g r"   )_default_store)stores    r)   _set_default_storer5   ]   s    Nr(   c            	         [         [        ;  aH  [        5       n [        U R                  S[        [        U R                  5      5      5      [        [         '   [        $ )Nr   )_global_env_gid
_group_mapr*   r
   ranklistrange
world_size)genvs    r)   _get_group_mapr>   b   sB    j( &+IIq$uT__56'

?# r(   c                 $    [        5       [           $ r"   )r>   r7   r'   r(   r)   _get_global_groupr@   l   s    O,,r(   c                     [         $ r"   _group_map_by_namer'   r(   r)   _get_group_map_by_namerD   p   s    r(   c                 P    [        5       (       d   S5       e[        5       [           $ )NzZCall paddle.distributed.init_parallel_env first to initialize the distributed environment.)r   rD   _default_group_namer'   r(   r)   _get_default_grouprG   u   s.     	5 "#$788r(   c                .    U [         ;  d   eU[         U '   g r"   )r8   )gidgroups     r)   _set_group_maprK   ~   s    j   JsOr(   c                .    U [         ;  d   eU[         U '   g r"   rB   )namerJ   s     r)   _set_group_map_by_namerN      s    ))))$tr(   c                .    U [         ;  d   eU[         U '   g r"   )_group_map_backend)rJ   r0   s     r)   _set_group_map_backendrQ      s    **** 'ur(   c                     [        5       (       a.  [        S-  q[        [        [        5       R                  S5      -   $ [        [        5       5      [        [        5       R                  S5      -   $ )Nr	   	   )r   _start_ring_idmaxr*   nringslenr>   r'   r(   r)   _new_ring_idrX      sS    !O$5$<$<a @@@>#$s?+<+C+CQ'GGGr(   c	           	     $   S n	[        5       n
U [        ;   d   SU  S35       eU S:X  a"  [        R                  R	                  XX65      n	U	$ U S:X  a1  [        R
                  R	                  UUUUU
R                  UU5      n	U	$ U S:X  a-  [        R                  R	                  XR                  X#U5      n	U	$ U S:X  a"  [        R                  R	                  XX65      n	U	$ U S:X  a.  [        R                  R	                  UUUUU
R                  U5      n	U	$ )NzUnsupported backend: .r   r   r   r   r   )r*   _valid_backend_listr   ProcessGroupGloocreateProcessGroupNCCL
pg_timeoutProcessGroupCustomdevice_typeProcessGroupBKCLProcessGroupFlagcx)r0   r4   r9   r<   
group_name
pg_optionsgroup_idnccl_comm_init_optionnccl_configpgr=   s              r)   _new_process_group_implrj      s0    
BD))M-B7)1+MM)&""))%zL4 I3 
F	""))OO!
0 I 
F	$$++##Tx
 I 
F	""))%zL I 
H	$$++OO!
 Ir(   c                    U q g r"   )_custom_gid)rI   s    r)   _set_custom_gidrm      s    Kr(   c                   [        5       (       Ga  [        (       a  [        O	[        5       n[        [	        U5      -   nUS:w  ac  U b  [        U 5      S:  aQ  [        5       nUR                  nUR                  n	Uc  [        OUnU c  U	n [        U 5      [        U	5      ::  d   S5       e[        U 5      n
[        U 5      n U
S:  a5  WU ;   a/  US:X  a  SOU R                  U5      n[        U[        UU
USUUUS9	nOSnSn[        XXUS9nU[        U'   U[         U'   U["        U'   [%        U5        U$ U(       d  S	nUS	:X  d   S
5       e['        5       nUR                  n[        5       nX;  a  [        SX5      nU[         U'   GOr[        U 5      n U R                  U5      n[        U 5      n[        UX5      nU[         U'   US:  Ga-  [(        R*                  " 5       nUUl        UUl        U  Vs/ s H  nUR0                  U   PM     snUl        UR2                  Ul        SUl        [(        R6                  " 5       (       aG  [(        R8                  " UR:                  5      n[(        R<                  " UU5      R?                  U5        On[(        R@                  " 5       (       aG  [(        RB                  " UR:                  5      n[(        RD                  " UU5      R?                  U5        O[G        S5      eU$ [        5       (       a  [H        RJ                  " S/SS9O[H        RL                  " S/SSS9n[H        RN                  RQ                  USS9  [H        RN                  RS                  U5        U$ s  snf )a  

Creates a new distributed communication group.

Args:
    ranks (list): The global ranks of group members.
    backend (str): The backend used to create group, only nccl is supported now.
    timeout (datetime.timedelta, optional): The waiting timeout for store relevant options, default is 30 minutes.

Returns:
    Group: The group instance.

Examples:
    .. code-block:: python

        >>> # doctest: +REQUIRES(env: DISTRIBUTED)
        >>> import paddle

        >>> paddle.distributed.init_parallel_env()
        >>> tindata = paddle.randn(shape=[2, 3])
        >>> gp = paddle.distributed.new_group([2, 4, 6])
        >>> paddle.distributed.all_reduce(tindata, group=gp, sync_op=False)

r,   Nr	   zQSize of new group must be less than or equal to that of the default global group.r   )re   rf   rg   rh   )ri   rM   r   z,backend other than nccl is not supported yet   zno cuda device foundint32)dtypeT)sync_op)*r   rl   rX   rF   strrW   rG   r9   ranksr/   sortedindexrj   r3   r
   rC   r8   rP   r   r*   r   ParallelStrategynranks
local_ranktrainer_endpointscurrent_endpointrV   is_compiled_with_cuda	CUDAPlace	device_idNCCLParallelContextinit_with_ring_idis_compiled_with_xpuXPUPlaceBKCLParallelContextAssertionErrorr$   	to_tensorfullr%   
all_reducewait)ru   r0   timeoutrg   rh   rI   rd   global_groupglobal_rankglobal_rankssizer9   ri   rJ   r=   ring_idgp
group_rank
group_sizestrategyiplacetmps                          r)   	new_groupr      s&   B ([kln(3s83
g5=CJN-/L&++K'--L*1/&wG}$u:\!22 =2 5zu!8u,7*1K0HD(&;'
B DBdJ?).:&
3$+5! 	ufLLLD))KnG2w& 
7u[[-
Z
:w. 
7?,,.H(HO",H38*38a&&q)5*H& )-(=(=H%HO))++t~~6((59KK **,,dnn5((59KK %%;<<I  	!G,[[!aw/ 
 !!#t!4
C I=*s   +M.c                 ,    [         R                  " 5       $ )a  
Check whether the distributed package is available.

Returns:
    Returns True if the distributed package is available, otherwise False.

Examples:
    .. code-block:: python

        >>> import paddle
        >>> print(paddle.distributed.is_available())

)r   is_compiled_with_distr'   r(   r)   is_availabler   T  s     %%''r(   c                $   [         R                  " 5       n[        5       nUR                  nUR                  nUR
                  nU S:X  a"  [         R                  R                  USX45        g U S:X  a  SnUR                   H  nXg-  nM	     USR                  S5      -  n[        R                  " UR                  SS95      R                  5       n[         R                  R                  U5        [         R                  R                  USX4U5        g U S:X  aS  UR                   n	["        R$                  R'                  U	 S	U 35        [         R                  R)                  USX4U	5        g U S
:X  a  SnUR                   H  nXg-  nM	     USR                  S5      -  n[        R                  " UR                  SS95      R                  5       n[         R                  R                  U5        [         R                  R+                  USX4U5        g g )Nr   0r    z
ring_id:{}zUTF-8)encodingr   :r   )r   create_or_get_global_tcp_storer*   r9   r<   r   CommContextManagercreate_gloo_comm_contextr{   formathashlibmd5encode	hexdigestset_device_idcreate_nccl_comm_contextra   r$   device
set_devicecreate_xccl_comm_contextcreate_bkcl_comm_context)
r0   r4   
global_envr9   r<   dev_idendpoints_strendpointendpoints_str_hashdev_types
             r)   _init_parallel_envr   e  s   //1E "J??D&&J!!F&883	
 
F	"44H%M 5,,S11$[[  ' 2

)+ 	 	--f5883*<	
 
F	))  H:Qvh!78883(	
 
F	"44H%M 5,,S11$[[  ' 2

)+ 	 	--f5883*<	
 
r(   c                     [         $ r"   _shutdown_group_map_by_namer'   r(   r)   _get_shutdown_group_map_by_namer     s    &&r(   c                    U[         U '   g r"   r   )pg_namerJ   s     r)   "_update_shutdown_group_map_by_namer     s    +0(r(   c                    [         U 	 g r"   r   )r   s    r)   "_delete_shutdown_group_map_by_namer     s
    #G,r(   c                 ,    [         R                  5         g r"   )r   clearr'   r(   r)   !_clear_shutdown_group_map_by_namer     s    %%'r(   c                   [        5       nU ci  [        5       R                  5        HL  u  p#UR                  c  M  X!;  d  M  U[        :w  d  M'  UR                  R                  5         [        X#5        MN     g U R                  bB  U R                  U;  a1  U R                  R                  5         [        U R                  U 5        g g g r"   )r   rD   itemsprocess_grouprF   shutdownr   rM   )rJ   shutdown_groupsr   ri   s       r)   shutdown_process_groupr     s    57O}1399;KG  ,222  ))+27? < +

/1((*.uzz5A 2 ,r(   c                2   [        5       nU c<  UR                  5        H  nUR                  R                  5         M     [	        5         g U R                  bA  U R
                  U;   a0  U R                  R                  5         [        U R
                  5        g g g r"   )r   valuesr   restartr   rM   r   )rJ   r   ri   s      r)   restart_process_groupr     sx    57O}!((*B$$& +)+*uzz_/L'').uzz: 0M*r(   )r   r   N)ru   zlist[int] | Noner0   zLiteral['nccl'] | Noner   zdatetime.timedeltarg   intrh   zNCCLConfig | Nonereturnr
   )r   bool)r0   r   r   Noner"   )rJ   zGroup | Noner   r   )I
__future__r   datetimer   typingr   r   typing_extensionsr   r$   paddle.baser   paddle.frameworkr   communication.groupr
   r   r   fleet.layers.mpu.mp_opsr   r   r   r   r   r   r   r   r   r   r   r   r   r   __annotations__paddle.base.libpaddler    __all__r#   r*   r8   r7   rC   rP   rF   r[   r3   r/   	timedelta_default_timeoutrT   r1   r5   r>   r@   rD   rG   rK   rN   rQ   rX   rj   rl   rm   r   r   r   r   r   r   r   r   r   r   r'   r(   r)   <module>r      s   #  
 (   , F F     %&NOL)O0
 
     $ I  %%d3 

-
9%(H" )^  #&*"2!"%)@@#@  @ 	@
 #@ @F("(
V ! '
1
-
(
B,
;r(   