
    ΑiV                    h   S SK Jr  S SKrS SKrS SKJrJrJrJr  S SK	r
S SKrS SKJr  S SKJr  S SKJr  S SKJr  \R(                  " \5      r\(       a$  S SKJrJr  S S	KJr  S SKJr  S S
KJr  \\\\   4   r Sq!Sq"0 q#0 q$S r%S r&S r'S r(S r)S r*S r+S r, " S S\RZ                  5      r-S r.S r/g)    )annotationsN)TYPE_CHECKINGAnySupportsIndexUnion)fleet)_get_group_map)is_initialized)core)IterableSequence)TracebackType)NestedNumericSequencec                     [         $ N)_g_current_process_mesh     m/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/auto_parallel/process_mesh.pyget_current_process_meshr   2   s    ""r   c                    [         qU q g r   )r   _g_previous_process_mesh)process_meshs    r   set_current_process_meshr   7   s      7*r   c                     [         qg r   )r   r   r   r   r   reset_current_process_meshr   >   s
     7r   c                v    SU  SU 3nU[         ;   a  [         U   nU$ [        [         5      S-   nU[         U'   U$ )Nshape , process_ids    )_g_unique_process_mesh_maplen)shapeprocess_idskey	unique_ids       r   get_unique_id_for_process_meshr'   D   sR    5'}
5C
((.s3	
  23a7	*3"3'r   c                <    SU  SU 3nU[         ;   d   e[         U   $ )Nr   r   r!   )r#   r$   r%   s      r   #retrieve_unique_id_for_process_meshr*   P   s.    5'}
5C,,,,%c**r   c                     [         $ r   r)   r   r   r   get_unique_process_mesh_mapr,   W   s    %%r   c                    U c  / n [        U [        5      (       d   S5       eU  H  nU[        ;   a  M  0 [        U'   M     g )Nzdim_names must be a list.)
isinstancelist_g_group_map)	dim_namesdim_names     r   init_group_by_process_meshr3   \   sE    	i&&C(CC&|#!#X r   c                D    U [         ;  a  [        SU  35      e[         U    $ )NzNo group found for dim_name )r0   RuntimeError)r2   s    r   get_group_map_by_dim_namer6   g   s(    |#9(DEE!!r   c                    ^  \ rS rSr% SrS\S'   S\S'       S         SS jjr\SS j5       r\SS j5       r	\SS	 j5       r
    SS
 jr      SS jrS S jr S!     S"S jjr    S#S jr S!   S$S jjr S!   S$S jjrS%S jr        S&S jrS'S jrS(S jrS)S jrS*S jrSU 4S jjrSrU =r$ )+ProcessMeshn   a[  
The `ProcessMesh` object describes the Cartesian topology of the used processes.

Args:
    mesh (list|numpy.array): an n-dimensional array describes the topology
        of the processes.
    dim_names (list, optional): the i-th element of this list gives the name of the
        i-th dimension of the mesh.

Examples:
    .. code-block:: python

        >>> import paddle
        >>> import paddle.distributed as dist

        >>> mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]], dim_names=["x", "y"])
        >>> assert mesh.shape == [2, 3]
        >>> assert mesh.process_ids == [2, 4, 5, 0, 1, 3]

z	list[int]r#   r$   c                   S[         R                  R                  R                  l        Uc/  Uc   eUc   e[
        R                  " U5      R                  U5      n[        U[        5      (       d*  [        U[
        R                  5      (       d  [        S5      e[        U[        5      (       a  [
        R                  " U5      nUb   [        U[        5      (       d  [        S5      eXl        [        U R                  R                  5      U l        U R                  R                  5       R!                  5       U l        [%        S U R"                   5       5      (       d   S5       e['        U R"                  5      S:  d   S5       e[)        U R"                  5      n[+        U5      [+        U R"                  5      :X  d   S5       eUbE  [+        U5      [+        U R                  5      :X  d   S	5       e[,        R.                  " U5      U l        O@[3        [+        U R                  5      5       Vs/ s H  nS
[5        U5      -   PM     snU l        [)        U R0                  5      n[+        U5      [+        U R0                  5      :X  d   SU S35       e[6        R8                  R;                  X R                  U R"                  U R0                  5        SSKJn  U" 5       n	U	RA                  U 5        SSK!J"n
  U
" S5      nURG                  U RH                  5        [K        U R                  U R"                  5      U l&        [O        U R0                  5        g s  snf )NTz3The mesh must be an instance of list or np.ndarray.z*The dim_names must be an instance of list.c              3  B   #    U  H  n[        U[        5      v   M     g 7fr   )r.   int).0ps     r   	<genexpr>'ProcessMesh.__init__.<locals>.<genexpr>   s     A/@!:a%%/@s   z(All elements of the mesh must be integerr   z&All elements of the mesh must be >= 0.z(All elements of the mesh must be unique.z?The length of dims_names must be same as the shape of the mesh.dzAll dim_names z must be unique.r    get_default_distributed_context)get_process_group)(paddlebase	framework
global_var_in_auto_parallel_nparrayreshaper.   r/   ndarray
ValueError_meshr#   _shapeflattentolist_process_idsallminsetr"   copydeepcopy
_dim_namesrangestrr   r8   __init__static.dist_contextrC   add_process_meshstatic.process_grouprD   	add_ranksr$   r'   
_unique_idr3   )selfmeshr1   r#   r$   unique_process_idsiunique_dim_namesrC   default_dist_cxtrD   pg0s               r   r\   ProcessMesh.__init__   s    ?C((; <$$$***88K(007D$%%jrzz.J.JE  dD!!88D>D It)D)DIJJ
4::++, JJ..0779At/@/@AAA 	
6	
A 4$$%* 	
4	
* !!2!23%&#d.?.?*@@ 	
6	
@  y>S%55 Q5 #mmI6DO5:3t{{;K5LM5LsSV|5LMDOt/#$DOO(<< 	
YK'78	
<
 	!!++t00$//	

 	I:<))$/;"d&&' 9KK**
 	#4??35 Ns   .Mc                    U R                   $ )z)
Get the underlying mesh of ProcessMesh.
)rO   rb   s    r   rc   ProcessMesh.mesh   s    
 zzr   c                    U R                   $ )z4
Get the underlying dimension names of ProcessMesh.
)rY   rk   s    r   r1   ProcessMesh.dim_names   s    
 r   c                    U R                   $ )z
Get the unique id of ProcessMesh.
NOTE
Unique id only take process_ids and shape into account.
Different ProcessMesh with same process_ids and shape have same unique id.
)ra   rk   s    r   r&   ProcessMesh.unique_id   s     r   c                ~   [        U[        5      (       a  / n[        U5       H:  u  p4[        U[        5      (       d  M  UR	                  U R
                  U   5        M<     U R                  U   nUR                  (       a  [        XR5      $ [        U/5      $ [        U[        5      (       a&  U R                  U   nU R
                  n[        XR5      $ [        U[        5      (       a  U R                  U5      $ U R                  U   nU R
                  SS  nUR                  (       a  [        XR5      $ [        U/5      $ )Nr    )r.   tuple	enumeratesliceappendrY   rO   r#   r8   r[   get_submesh_with_dim)rb   indexnew_dim_namesre   itemnew_meshs         r   __getitem__ProcessMesh.__getitem__   s    eU##M$U+dE**!(();< , zz%(H~~"8;; #H:..u%%zz%(H OOMx77s##,,U33zz%(H OOAB/M~~"8;;"H:..r   c                N   X R                   ;  a  gUc  g[        U[        5      (       a  U R                  U   nO#[        U[        5      (       a  UnO[        S5      eU R                  R                  U5      n[        [        R                  " U R                  U:H  5      U   5      $ )Nr   #dim must be a string or an integer.)
rS   r.   r<   rY   r[   rN   rw   rJ   whererO   )rb   dim
process_idr2   dim_name_indexs        r   get_rank_by_dim_and_process_id*ProcessMesh.get_rank_by_dim_and_process_id  s     ...;c3s+HS!!HBCC..x8288DJJ*45nEFFr   c                   Uc  g[        U[        5      (       a  U R                  U   nO#[        U[        5      (       a  UnO[	        S5      eX R                  ;   d   eU R
                  U R                  R                  U5         $ )Nr    r   )r.   r<   rY   r[   rN   rP   rw   )rb   r   r2   s      r   get_dim_sizeProcessMesh.get_dim_size  sq    ;c3s+HS!!HBCC??***{{4??00:;;r   c                   XR                   ;   d
   U S35       eU R                   R                  U5      nU/[        [        U R                   5      5       Vs/ s H  oDU:w  d  M
  UPM     sn-   nU/U R                    Vs/ s H  ofU:w  d  M
  UPM     sn-   nU R                  R                  U5      nUb2  [        USS  5      S:  a  [        X   USS  5      $ [        X   /U5      $ [        X5      $ s  snf s  snf )N is not a valid dim name.r    r   )rY   rw   rZ   r"   rO   	transposer8   )	rb   r2   rw   
index_axisre   	new_orderr   rx   rz   s	            r   get_mesh_with_dimProcessMesh.get_mesh_with_dim(  s   
 ??* 	
j12	
* __**84
LS12$
2!:oA2$
 
	 "
??&
*CXoC?&
 
 ::''	2=$%)"8?M!"4EFF #HO#4mDD833$
&
s   	C2$C2?	C7C7c                   U R                  U5      R                  R                  U R                  U5      S5      n[        R
                  R                  5       nX0R                  ;  a  [        R                  SU S35        g[        R                  " X#:H  5      UR                  S   -  n[        USS2U4   U/5      nU$ )a  
Slice the current ProcessMesh based on the dim_name given to create a submesh with single dimension remained.

Args:
    dim_name (str): the name of the mesh dimension of the ProcessMesh to create the submesh for.
Returns:
    A :class:`ProcessMesh` object

Examples:
    .. code-block:: python

        >>> import paddle
        >>> import paddle.distributed as dist

        >>> dist.init_parallel_env()
        >>> mesh_2d = dist.ProcessMesh([[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "tp"])

        >>> dp_mesh = mesh_2d.get_submesh_with_dim("dp")
        >>> # ProcessMesh:([0, 4]) on rank 0, 4
        >>> # ProcessMesh:([1, 5]) on rank 1, 5
        >>> # ProcessMesh:([2, 6]) on rank 2, 6
        >>> # ProcessMesh:([3, 7]) on rank 3, 7

        >>> tp_mesh = mesh_2d.get_submesh_with_dim("tp")
        >>> # ProcessMesh:([0, 1, 2, 3]) on rank 0, 1, 2, 3
        >>> # ProcessMesh:([4, 5, 6, 7]) on rank 4, 5, 6, 7

        >>> mesh_3d = dist.ProcessMesh([[[0, 1],[2, 3]], [[4, 5], [6, 7]]], dim_names=["pp","dp","tp"])

        >>> pp_mesh = mesh_3d.get_submesh_with_dim("pp")
        >>> # ProcessMesh:([0, 4]) on rank 0, 4
        >>> # ProcessMesh:([1, 5]) on rank 1, 5
        >>> # ProcessMesh:([2, 6]) on rank 2, 6
        >>> # ProcessMesh:([3, 7]) on rank 3, 7

        >>> dp_mesh = mesh_3d.get_submesh_with_dim("dp")
        >>> # ProcessMesh:([0, 2]) on rank 0, 2
        >>> # ProcessMesh:([1, 3]) on rank 1, 3
        >>> # ProcessMesh:([4, 6]) on rank 4, 6
        >>> # ProcessMesh:([5, 7]) on rank 5, 7

        >>> tp_mesh = mesh_3d.get_submesh_with_dim("tp")
        >>> # ProcessMesh:([0, 1]) on rank 0, 1
        >>> # ProcessMesh:([2, 3]) on rank 2, 3
        >>> # ProcessMesh:([4, 5]) on rank 4, 5
        >>> # ProcessMesh:([6, 7]) on rank 6, 7
r~   zRank z- is not in the process mesh, just return NoneN)r   rO   rL   r   rE   distributedget_rankrS   loggerwarningrJ   argmaxr#   r8   )rb   r2   reorder_mesh	curr_rankcol_idxsub_meshs         r   rv    ProcessMesh.get_submesh_with_dimA  s    h --h7==EEh'
 &&//1	---NN	{"OP ))L569K9KB9OO|AwJ7(Dr   c                r   [        5       (       d   S5       e[        U R                  5      S:  a  Uc  [        S5      eU R	                  U5      R
                  R                  U R                  U5      S5      n[        R                  R                  5       n[        U5      nU R                   H\  n[        R                  " X%:H  5      UR                  S   -  nXd;   a  M2  [        R                  R!                  USS2U4   5      nXtU'   M^     [        R                  " X#:H  5      UR                  S   -  nXH   $ ) When you want to get a group from the ProcessMesh. Call paddle.distributed.init_parallel_env first to initialize the distributed environment.r    NRYou should specify the dim_name when the ProcessMesh has more than one dimensions.r~   )r
   r"   rY   rN   r   rO   rL   r   rE   r   r   r6   rS   rJ   r   r#   	new_group)	rb   r2   r   r   groupsrankr   pgcur_col_idxs	            r   
_get_groupProcessMesh._get_group  s+   
  	
9	

 t!#(8d  --h7==EEh'
 &&//1	*84%%Dii 458J8J28NNG ##--l1g:.FGB 7O & IIl/0<3E3Eb3II 	 ""r   c                   [        5       (       d   S5       e[        U R                  5      S:  a  Uc  [        S5      e[        U R                  5      S:X  Ga3  Ub)  XR                  ;  a  [        U SU R                   35      e[	        [
        R
                  S5      (       an  [
        R                  " 5       nUbV  UR                  UR                  UR                  UR                  UR                  S.nX;  a  [        U S35      eX1   " 5       $ [        5       nUR                  5        H3  n[        UR                  5      [        U R                   5      :X  d  M1  Us  $    ["        R$                  R'                  U R                   5      $ XR                  ;  a  [        U SU R                   35      eU R)                  U5      nUR+                  U5      $ )z
Convert single dimension ProcessMesh to the corresponding Group.

Args:
    dim_name (str, optional): it can be the name of the mesh dimension. Default is None.

Returns:
    A :class:`Group` object.
r   r    r   z not in the dimension names _hcg)ppdpmpsepshardingr   )r
   r"   rY   rN   hasattrr   get_hybrid_communicate_groupget_pipe_parallel_groupget_data_parallel_groupget_model_parallel_groupget_sep_parallel_groupget_sharding_parallel_groupr	   valuesrV   ranksrS   rE   r   r   rv   	get_group)rb   r2   hcgparallel_group_map	group_mapgroupr   s          r   r   ProcessMesh.get_group  s     	
9	

 t!#(8d  t1$#(G j <T__<MN  5;;//<<>C"%"="="%"="="%">">#&#=#=(+(G(G.* $=",#+*,E F#   2;==*,	&--/E5;;'3t/@/@+AA$ 0 ))33D4E4EFF. j <T__<MN  00:H%%h//r   c                    [        U 5        [        R                  R                  5       nUR	                  5       n[        UR                  R                  5       5      U l        [        UR                  5      U l        g r   )r   rE   staticdefault_main_programcurrent_blockr/   varskeys_old_var_namesr"   ops_old_op_size)rb   default_prog	cur_blocks      r   	__enter__ProcessMesh.__enter__  sU     &}}99; ..0	"9>>#6#6#89	.r   c                V   SSK Jn  SSKJn  [        R
                  R                  5       nUR                  5       n[        UR                  R                  5       5      n[        UR                  5      n	SSKJn
  U
" 5       nU H  nXR                  ;  d  M  UR                  U   nUR!                  U5      nUcS  U" UR                  U   5      nXR"                  l        UR"                  R'                  S5        UR)                  U5        M  UR"                  R$                  b  M  XR"                  l        UR"                  R'                  S5        M     [+        U R,                  U	5       H  nUR                  U   nUR/                  U5      nUcG  U" U5      nU UR"                  l        UR"                  R'                  S5        UR1                  U5        Mm  UR"                  R$                  b  M  U UR"                  l        UR"                  R'                  S5        M     [3        5         g )Nr    )DistributedOperator)DistributedTensorrB   r   )static.dist_opr   static.dist_tensorr   rE   r   r   r   r/   r   r   r"   r   r]   rC   r   get_dist_tensor_for_program	dist_attrr   mark_annotatedadd_dist_tensor_for_programrZ   r   get_dist_op_for_programadd_dist_op_for_programr   )rb   exc_type	exc_value	tracebackr   r   r   r   new_var_namesnew_op_sizerC   default_dist_ctxnametensordist_tensoridxopdist_ops                     r   __exit__ProcessMesh.__exit__  s    	89}}99; ..0	Y^^0023)--(H:<!D..."-.JJ &"3INN44H"IK9=))6))88H$@@M",,99A=A--:#--<<^L "  **K8Cs#B&>>rBG-b115!!.!!00@ 88A$$11959G%%2%%44^D 9 	#$r   c                    [        U 5      U;   a  U[        U 5         $ [        [        R                  " U R                  5      U R
                  5      nX![        U 5      '   U$ r   )idr8   rJ   rK   rc   r1   )rb   memonew_process_meshs      r   __deepcopy__ProcessMesh.__deepcopy__  sK    d8t4>!&rxx		':DNNK)RXr   c                    [        U[        [        R                  45      (       d  gU R                  UR                  :w  d  U R                  UR                  :w  a  gg)NFT)r.   r8   r   r#   r$   rb   others     r   __eq__ProcessMesh.__eq__  sF    %+t/?/?!@AA::$(8(8E<M<M(Mr   c                .    U R                  U5      (       + $ r   )r   r   s     r   __ne__ProcessMesh.__ne__"  s    ;;u%%%r   c                X    SU R                    SU R                   SU R                   3nU$ )Nr   r   z, dim_names )r#   r$   r1   )rb   r[   s     r   __str__ProcessMesh.__str__%  s/    tzzl.1A1A0B,t~~N^_
r   c                    > [         TU ]  5       $ r   )super__hash__)rb   	__class__s    r   r   ProcessMesh.__hash__)  s    w!!r   )rY   rO   r   r   rS   rP   ra   )NNNN)
rc   z/npt.NDArray[Any] | NestedNumericSequence | Noner1   zlist[str] | Noner#   z_NumpyShapeLike | Noner$   zIterable[Any] | NonereturnNone)r   znpt.NDArray[Any])r   z	list[str])r   r<   )rw   z/slice | tuple[slice, ...] | str | SupportsIndexr   r8   )r   	str | intr   r<   r   r<   )r   r   r   r<   r   )r2   r[   rw   z0slice | tuple[slice, ...] | SupportsIndex | Noner   r8   )r2   r[   r   r8   )r2   z
str | Noner   z,paddle.distributed.communication.group.Group)r   r   )r   ztype[BaseException] | Noner   zBaseException | Noner   zTracebackType | Noner   r   )r   r   r   r8   )r   ProcessMesh | core.ProcessMeshr   bool)r   r   r   r   )r   r[   )__name__
__module____qualname____firstlineno____doc____annotations__r\   propertyrc   r1   r&   r{   r   r   r   rv   r   r   r   r   r   r   r   r   r   __static_attributes____classcell__)r   s   @r   r8   r8   n   s   *  AE&*(,,0I4=I4 $I4 &	I4
 *I4 
I4V      /D/	/8GG*-G	G*<  CG44 @4 
	42@@ 
@H  $## 
6#D  $:0:0 
6:0x/,%,,% (,% (	,%
 
,%\ &" "r   r8   c                    U (       d  gS nSnU  H  nU" X#5      u  pBU(       a  M    g   [         R                  " U5      $ )zCCompute the compatible process mesh given a list of process meshes.Nc                   U c  SU4$ Uc  SU 4$ X:X  a  SU 4$ U R                   UR                   :X  a4  [        U R                  5      [        UR                  5      :  a  SU 4$ SU4$ [        U R                   5      n[        UR                   5      nUR	                  U5      (       a  SU4$ UR	                  U5      (       a  SU 4$ g)NT)FN)r$   r"   r#   rV   issubset)pm1pm2process_set1process_set2s       r   '_compute_compatible_process_mesh_of_twoPcompute_compatible_process_mesh.<locals>._compute_compatible_process_mesh_of_two2  s    ;9;9:9??coo-399~SYY/Sy Sy 3??+3??+  ..9  ..9r   )rW   rX   )process_mesh_listr  compatible_resultr   
compatibles        r   compute_compatible_process_meshr  -  sK    ( )(O)
%
 z * ==*++r   c                    Sn[        5       nU  H.  nUc  M  [        UR                  5      nUR                  U5      nM0     [        U5      S:w  a  [	        [        U5      5      nU$ )zMerge a list of process meshes.Nr   )rV   r$   unionr"   r8   r/   )process_meshesmerged_process_meshmerged_process_idsr   r$   s        r   merge_process_meshesr  P  sh    &#l667K!3!9!9+!F ' !#)$/A*BCr   )0
__future__r   rW   loggingtypingr   r   r   r   numpyrJ   rE   paddle.distributedr   paddle.distributed.collectiver	   &paddle.distributed.communication.groupr
   paddle.frameworkr   	getLoggerr   r   collections.abcr   r   typesr   numpy.typingnptpaddle._typingr   _NumpyShapeLiker   r   r!   r0   r   r   r   r'   r*   r,   r3   r6   r8   r  r  r   r   r   <module>r(     s    #   ; ;   $ 8 A !			8	$2#4M8M+BBCO     #
+7	+&
$"|"$"" |"~ ,F
r   