
    x-jV                    d   d dl mZ d dlZd dlZd dlmZmZmZmZ d dl	Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ  ej        e          Zer*d dlmZmZ d d	lmZ d dlmZ d d
lmZ eeee         f         Z da!da"i a#i a$d Z%d Z&d Z'd Z(d Z)d Z*d Z+d Z, G d dej-                  Z-d Z.d Z/dS )    )annotationsN)TYPE_CHECKINGAnySupportsIndexUnion)fleet)_get_group_map)is_initialized)core)IterableSequence)TracebackType)NestedNumericSequencec                     t           S N)_g_current_process_mesh     m/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/auto_parallel/process_mesh.pyget_current_process_meshr   2   s    ""r   c                    t           a| a d S r   )r   _g_previous_process_mesh)process_meshs    r   set_current_process_meshr   7   s      7*r   c                     t           ad S r   )r   r   r   r   r   reset_current_process_meshr   >   s     7r   c                    d|  d| }|t           v rt           |         }n!t          t                     dz   }|t           |<   |S )Nshape , process_ids    )_g_unique_process_mesh_maplen)shapeprocess_idskey	unique_ids       r   get_unique_id_for_process_meshr'   D   sS    
55
5
5
5
5C
(((.s3		233a7	*3"3'r   c                B    d|  d| }|t           v sJ t           |         S )Nr   r   r!   )r#   r$   r%   s      r   #retrieve_unique_id_for_process_meshr*   P   s4    
55
5
5
5
5C,,,,,%c**r   c                     t           S r   r)   r   r   r   get_unique_process_mesh_mapr,   W   s    %%r   c                ~    | g } t          | t                    s
J d            | D ]}|t          v ri t          |<   d S )Nzdim_names must be a list.)
isinstancelist_g_group_map)	dim_namesdim_names     r   init_group_by_process_meshr3   \   s`    	i&&CC(CCC& $ $|##!#X$ $r   c                R    | t           vrt          d|            t           |          S )NzNo group found for dim_name )r0   RuntimeError)r2   s    r   get_group_map_by_dim_namer6   g   s/    |##D(DDEEE!!r   c                      e Zd ZU dZded<   ded<   	 	 	 	 d8d9dZed:d            Zed;d            Zed<d            Z	d=dZ
d>dZd?dZ	 d@dAd ZdBd!Z	 d@dCd$Z	 d@dCd%ZdDd&ZdEd-ZdFd0ZdGd4ZdHd5ZdId6Zd< fd7Z xZS )JProcessMesha  
    The `ProcessMesh` object describes the Cartesian topology of the used processes.

    Args:
        mesh (list|numpy.array): an n-dimensional array describes the topology
            of the processes.
        dim_names (list, optional): the i-th element of this list gives the name of the
            i-th dimension of the mesh.

    Examples:
        .. code-block:: python

            >>> import paddle
            >>> import paddle.distributed as dist

            >>> mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]], dim_names=["x", "y"])
            >>> assert mesh.shape == [2, 3]
            >>> assert mesh.process_ids == [2, 4, 5, 0, 1, 3]

    z	list[int]r#   r$   Nmesh/npt.NDArray[Any] | NestedNumericSequence | Noner1   list[str] | None_NumpyShapeLike | NoneIterable[Any] | NonereturnNonec                h   dt           j        j        j        _        |/|J |J t          j        |                              |          }t          |t                    s)t          |t
          j
                  st          d          t          |t                    rt          j        |          }|$t          |t                    st          d          || _        t          | j        j                  | _        | j                                                                        | _        t%          d | j        D                       s
J d            t'          | j                  dk    s
J d            t)          | j                  }t+          |          t+          | j                  k    s
J d            |It+          |          t+          | j                  k    s
J d	            t-          j        |          | _        n0d
 t3          t+          | j                            D             | _        t)          | j                  }t+          |          t+          | j                  k    sJ d| d            t4          j                            | | j        | j        | j                   ddlm}  |            }|                    |            ddl m!}	  |	d          }
|
"                    | j#                   tI          | j        | j                  | _%        tM          | j                   d S )NTz3The mesh must be an instance of list or np.ndarray.z*The dim_names must be an instance of list.c              3  @   K   | ]}t          |t                    V  d S r   )r.   int).0ps     r   	<genexpr>z'ProcessMesh.__init__.<locals>.<genexpr>   s,      AA!:a%%AAAAAAr   z(All elements of the mesh must be integerr   z&All elements of the mesh must be >= 0.z(All elements of the mesh must be unique.z?The length of dims_names must be same as the shape of the mesh.c                2    g | ]}d t          |          z   S )d)str)rC   is     r   
<listcomp>z(ProcessMesh.__init__.<locals>.<listcomp>   s"    MMMsSVV|MMMr   zAll dim_names z must be unique.r    get_default_distributed_context)get_process_group)'paddlebase	framework
global_var_in_auto_parallel_nparrayreshaper.   r/   ndarray
ValueError_meshr#   _shapeflattentolist_process_idsallminsetr"   copydeepcopy
_dim_namesranger   r8   __init__static.dist_contextrL   add_process_meshstatic.process_grouprM   	add_ranksr$   r'   
_unique_idr3   )selfr9   r1   r#   r$   unique_process_idsunique_dim_namesrL   default_dist_cxtrM   pg0s              r   rd   zProcessMesh.__init__   s<    ?C(; <$$$***8K((0077D$%% 	jrz.J.J 	E   dD!! 	"8D>>D It)D)D IJJJ
4:+,, J..007799AAt/@AAAAA 	
 	
6	
 	
A 4$%%***4 +** !!233%&&#d.?*@*@@@@6 A@@  y>>S%5%5555Q 655 #mI66DOOMMU3t{;K;K5L5LMMMDOt//#$$DO(<(<<<<8Y888 =<<
 	!!$+t0$/	
 	
 	

 	IHHHHH::<<))$///;;;;;;""d&''' 9K*
 
 	#4?33333r   npt.NDArray[Any]c                    | j         S )z9
        Get the underlying mesh of ProcessMesh.
        )rX   rj   s    r   r9   zProcessMesh.mesh   s    
 zr   	list[str]c                    | j         S )zD
        Get the underlying dimension names of ProcessMesh.
        )rb   rq   s    r   r1   zProcessMesh.dim_names   s    
 r   rB   c                    | j         S )z
        Get the unique id of ProcessMesh.
        NOTE
        Unique id only take process_ids and shape into account.
        Different ProcessMesh with same process_ids and shape have same unique id.
        )ri   rq   s    r   r&   zProcessMesh.unique_id   s     r   index/slice | tuple[slice, ...] | str | SupportsIndexc                x   t          |t                    rg }t          |          D ]:\  }}t          |t                    r |                    | j        |                    ;| j        |         }|j        rt          ||          S t          |g          S t          |t                    r$| j        |         }| j        }t          ||          S t          |t                    r| 
                    |          S | j        |         }| j        dd          }|j        rt          ||          S t          |g          S )Nr    )r.   tuple	enumeratesliceappendrb   rX   r#   r8   rH   get_submesh_with_dim)rj   ru   new_dim_namesrI   itemnew_meshs         r   __getitem__zProcessMesh.__getitem__   s=    eU## 	/M$U++ = =4dE** =!(();<<<z%(H~ /"8];;; #H:...u%% 	/z%(H OMx777s## 	/,,U333z%(H OABB/M~ /"8];;;"H:...r   dim	str | int
process_idc                H   || j         vrdS |dS t          |t                    r| j        |         }n't          |t                    r|}nt          d          | j                            |          }t          t          j        | j	        |k              |                   S )Nr   #dim must be a string or an integer.)
r\   r.   rB   rb   rH   rW   ru   rS   whererX   )rj   r   r   r2   dim_name_indexs        r   get_rank_by_dim_and_process_idz*ProcessMesh.get_rank_by_dim_and_process_id  s     T...2;1c3 	Ds+HHS!! 	DHHBCCC..x8828DJ*455nEFFFr   c                    |dS t          |t                    r| j        |         }n't          |t                    r|}nt	          d          || j        v sJ | j        | j                            |                   S )Nr    r   )r.   rB   rb   rH   rW   rY   ru   )rj   r   r2   s      r   get_dim_sizezProcessMesh.get_dim_size  s    ;1c3 	Ds+HHS!! 	DHHBCCC4?****{4?00::;;r   r2   rH   0slice | tuple[slice, ...] | SupportsIndex | Nonec                   | j         v sJ  d            | j                                       gfdt          t          | j                             D             z   }gfd| j         D             z   }| j                            |          }|Pt          |dd                    dk    rt          ||         |dd                    S t          ||         g|          S t          ||          S )N is not a valid dim name.c                     g | ]
}|k    |S r   r   )rC   rI   
index_axiss     r   rJ   z1ProcessMesh.get_mesh_with_dim.<locals>.<listcomp>1  s%     $
 $
 $
a:ooAooor   c                     g | ]
}|k    |S r   r   )rC   r   r2   s     r   rJ   z1ProcessMesh.get_mesh_with_dim.<locals>.<listcomp>4  s%     &
 &
 &
cXooCooor   r    r   )rb   ru   rc   r"   rX   	transposer8   )rj   r2   ru   	new_orderr}   r   r   s    `    @r   get_mesh_with_dimzProcessMesh.get_mesh_with_dim(  s=   
 4?***222 +** _**844
L $
 $
 $
 $
S1122$
 $
 $
 
	 "
 &
 &
 &
 &
?&
 &
 &
 
 :''	22=$%%))"8E?M!""4EFFF #HUO#4mDDD8]333r   c                   |                      |          j                            |                     |          d          }t          j                                        }|| j        vr t          	                    d| d           dS t          j        ||k              |j        d         z  }t          |dd|f         |g          }|S )a  
        Slice the current ProcessMesh based on the dim_name given to create a submesh with single dimension remained.

        Args:
            dim_name (str): the name of the mesh dimension of the ProcessMesh to create the submesh for.
        Returns:
            A :class:`ProcessMesh` object

        Examples:
            .. code-block:: python

                >>> import paddle
                >>> import paddle.distributed as dist

                >>> dist.init_parallel_env()
                >>> mesh_2d = dist.ProcessMesh([[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "tp"])

                >>> dp_mesh = mesh_2d.get_submesh_with_dim("dp")
                >>> # ProcessMesh:([0, 4]) on rank 0, 4
                >>> # ProcessMesh:([1, 5]) on rank 1, 5
                >>> # ProcessMesh:([2, 6]) on rank 2, 6
                >>> # ProcessMesh:([3, 7]) on rank 3, 7

                >>> tp_mesh = mesh_2d.get_submesh_with_dim("tp")
                >>> # ProcessMesh:([0, 1, 2, 3]) on rank 0, 1, 2, 3
                >>> # ProcessMesh:([4, 5, 6, 7]) on rank 4, 5, 6, 7

                >>> mesh_3d = dist.ProcessMesh([[[0, 1],[2, 3]], [[4, 5], [6, 7]]], dim_names=["pp","dp","tp"])

                >>> pp_mesh = mesh_3d.get_submesh_with_dim("pp")
                >>> # ProcessMesh:([0, 4]) on rank 0, 4
                >>> # ProcessMesh:([1, 5]) on rank 1, 5
                >>> # ProcessMesh:([2, 6]) on rank 2, 6
                >>> # ProcessMesh:([3, 7]) on rank 3, 7

                >>> dp_mesh = mesh_3d.get_submesh_with_dim("dp")
                >>> # ProcessMesh:([0, 2]) on rank 0, 2
                >>> # ProcessMesh:([1, 3]) on rank 1, 3
                >>> # ProcessMesh:([4, 6]) on rank 4, 6
                >>> # ProcessMesh:([5, 7]) on rank 5, 7

                >>> tp_mesh = mesh_3d.get_submesh_with_dim("tp")
                >>> # ProcessMesh:([0, 1]) on rank 0, 1
                >>> # ProcessMesh:([2, 3]) on rank 2, 3
                >>> # ProcessMesh:([4, 5]) on rank 4, 5
                >>> # ProcessMesh:([6, 7]) on rank 6, 7
        r   zRank z- is not in the process mesh, just return NoneN)r   rX   rU   r   rN   distributedget_rankr\   loggerwarningrS   argmaxr#   r8   )rj   r2   reorder_mesh	curr_rankcol_idxsub_meshs         r   r|   z ProcessMesh.get_submesh_with_dimA  s    h --h77=EEh''
 
 &//11	D---NNP	PPP   4)LI5669KB9OO|AAAwJ7(DDr   
str | None,paddle.distributed.communication.group.Groupc                   t                      s
J d            t          | j                  dk    r|t          d          |                     |          j                            |                     |          d          }t          j	        
                                }t          |          }| j        D ][}t          j        ||k              |j        d         z  }||v r-t          j	                            |dd|f                   }|||<   \t          j        ||k              |j        d         z  }||         S ) When you want to get a group from the ProcessMesh. Call paddle.distributed.init_parallel_env first to initialize the distributed environment.r    NRYou should specify the dim_name when the ProcessMesh has more than one dimensions.r   )r
   r"   rb   rW   r   rX   rU   r   rN   r   r   r6   r\   rS   r   r#   	new_group)	rj   r2   r   r   groupsrankr   pgcur_col_idxs	            r   
_get_groupzProcessMesh._get_group  sZ   
  	
 	
9	
 	

 t!##(8d   --h77=EEh''
 
 &//11	*844% 	! 	!Di 4558J28NNG&  #--l111g:.FGGB F7OO Ili/00<3Eb3II 	 k""r   c                p   t                      s
J d            t          | j                  dk    r|t          d          t          | j                  dk    r|"|| j        vrt          | d| j                   t	          t
          j        d          r\t          j                    }|G|j        |j        |j	        |j
        |j        d}||vrt          | d           ||                     S t                      }|                                D ]0}t          |j                  t          | j                  k    r|c S 1t"          j                            | j                  S || j        vrt          | d| j                   |                     |          }|                    |          S )	z
        Convert single dimension ProcessMesh to the corresponding Group.

        Args:
            dim_name (str, optional): it can be the name of the mesh dimension. Default is None.

        Returns:
            A :class:`Group` object.
        r   r    Nr   z not in the dimension names _hcg)ppdpmpsepshardingr   )r
   r"   rb   rW   hasattrr   get_hybrid_communicate_groupget_pipe_parallel_groupget_data_parallel_groupget_model_parallel_groupget_sep_parallel_groupget_sharding_parallel_groupr	   valuesr_   ranksr\   rN   r   r   r|   	get_group)rj   r2   hcgparallel_group_map	group_mapgroupr   s          r   r   zProcessMesh.get_group  s     	
 	
9	
 	

 t!##(8d   t1$$#(G(G NNT_NN   5;// ><>>C"%"="%"="%">#&#=(+(G. .* $+===",#+ F F F# #   <1(;===*,,	&--// % %E5;''3t/@+A+AAA$ B)33D4EFFFt.. NNT_NN   00::H%%h///r   c                   t          |            t          j                                        }|                                }t          |j                                                  | _        t          |j
                  | _        d S r   )r   rN   staticdefault_main_programcurrent_blockr/   varskeys_old_var_namesr"   ops_old_op_size)rj   default_prog	cur_blocks      r   	__enter__zProcessMesh.__enter__  sj     &&&}99;; ..00	"9>#6#6#8#899	..r   exc_typetype[BaseException] | None	exc_valueBaseException | None	tracebackTracebackType | Nonec                   ddl m} ddlm} t          j                                        }|                                }t          |j	        
                                          }t          |j                  }	ddlm}
  |
            }|D ]}|| j        vr|j	        |         }|                    |          }|R ||j	        |                   }| |j        _        |j                            d           |                    |           |j        j        &| |j        _        |j                            d           t+          | j        |	          D ]}|j        |         }|                    |          }|G ||          }| |j        _        |j                            d           |                    |           m|j        j        &| |j        _        |j                            d           t3                       d S )Nr    )DistributedOperator)DistributedTensorrK   r   )static.dist_opr   static.dist_tensorr   rN   r   r   r   r/   r   r   r"   r   re   rL   r   get_dist_tensor_for_program	dist_attrr   mark_annotatedadd_dist_tensor_for_programrc   r   get_dist_op_for_programadd_dist_op_for_programr   )rj   r   r   r   r   r   r   r   new_var_namesnew_op_sizerL   default_dist_ctxnametensordist_tensoridxopdist_ops                     r   __exit__zProcessMesh.__exit__  s-    	877777999999}99;; ..00	Y^002233)-((HHHHHH::<<! 	M 	MD4..."-.JJ  &"3"3IN44H"I"IK9=K)6)88HHH$@@MMMM",9A=A-:#-<<^LLL*K88 	E 	ECs#B&>>rBBG--b1115!.!00@@@ 88AAAA$1959G%2%44^DDD"$$$$$r   memor   c                    t          |           |v r|t          |                    S t          t          j        | j                  | j                  }||t          |           <   |S r   )idr8   rS   rT   r9   r1   )rj   r   new_process_meshs      r   __deepcopy__zProcessMesh.__deepcopy__  sV    d88t4>!&rx	':':DNKK)RXXr   otherProcessMesh | core.ProcessMeshboolc                    t          |t          t          j        f          sdS | j        |j        k    s| j        |j        k    rdS dS )NFT)r.   r8   r   r#   r$   rj   r   s     r   __eq__zProcessMesh.__eq__  sJ    %+t/?!@AA 	5:$$(8E<M(M(M5tr   c                .    |                      |           S r   )r   r   s     r   __ne__zProcessMesh.__ne__"  s    ;;u%%%%r   c                :    d| j          d| j         d| j         }|S )Nr   r   z, dim_names )r#   r$   r1   )rj   rH   s     r   __str__zProcessMesh.__str__%  s+    _tz__1A__t~__
r   c                D    t                                                      S r   )super__hash__)rj   	__class__s    r   r   zProcessMesh.__hash__)  s    ww!!!r   )NNNN)
r9   r:   r1   r;   r#   r<   r$   r=   r>   r?   )r>   ro   )r>   rr   )r>   rB   )ru   rv   r>   r8   )r   r   r   rB   r>   rB   )r   r   r>   rB   r   )r2   rH   ru   r   r>   r8   )r2   rH   r>   r8   )r2   r   r>   r   )r>   r?   )r   r   r   r   r   r   r>   r?   )r   r   r>   r8   )r   r   r>   r   )r   r   r>   r?   )r>   rH   )__name__
__module____qualname____doc____annotations__rd   propertyr9   r1   r&   r   r   r   r   r|   r   r   r   r   r   r   r   r   r   __classcell__)r   s   @r   r8   r8   n   s"         *  AE&*(,,0I4 I4 I4 I4 I4V    X    X    X/ / / /8G G G G*< < < <  CG4 4 4 4 42@ @ @ @H  $# # # # #D  $:0 :0 :0 :0 :0x/ / / /,% ,% ,% ,%\          & & & &   " " " " " " " " " "r   r8   c                n    | sdS d }d}| D ]} |||          \  }}|s dS t          j        |          S )zCCompute the compatible process mesh given a list of process meshes.Nc                j   | d|fS |d| fS | |k    rd| fS | j         |j         k    r2t          | j                  t          |j                  k    rd| fS d|fS t          | j                   }t          |j                   }|                    |          rd|fS |                    |          rd| fS dS )NT)FN)r$   r"   r#   r_   issubset)pm1pm2process_set1process_set2s       r   '_compute_compatible_process_mesh_of_twozPcompute_compatible_process_mesh.<locals>._compute_compatible_process_mesh_of_two2  s    ;9;9#::9?co--39~~SY//Sy Sy 3?++3?++  .. 	9  .. 	9{r   )r`   ra   )process_mesh_listr  compatible_resultr   
compatibles        r   compute_compatible_process_meshr  -  sy     t  ( )  (O(O|)
 )
%
%  	44	=*+++r   c                    d}t                      }| D ]-}|)t          |j                  }|                    |          }.t          |          dk    rt	          t          |                    }|S )zMerge a list of process meshes.Nr   )r_   r$   unionr"   r8   r/   )process_meshesmerged_process_meshmerged_process_idsr   r$   s        r   merge_process_meshesr  P  s    & G G#l677K!3!9!9+!F!F
!##)$/A*B*BCCr   )0
__future__r   r`   loggingtypingr   r   r   r   numpyrS   rN   paddle.distributedr   paddle.distributed.collectiver	   &paddle.distributed.communication.groupr
   paddle.frameworkr   	getLoggerr   r   collections.abcr   r   typesr   numpy.typingnptpaddle._typingr   _NumpyShapeLiker   r   r!   r0   r   r   r   r'   r*   r,   r3   r6   r8   r  r  r   r   r   <module>r"     s   # " " " " "   ; ; ; ; ; ; ; ; ; ; ; ;      $ $ $ $ $ $ 8 8 8 8 8 8 A A A A A A ! ! ! ! ! !		8	$	$ D22222222######444444M8M+BBCO     # # #
+ + +7 7 7	 	 	+ + +& & &
$ $ $" " "|" |" |" |" |"$" |" |" |"~ ,  ,  ,F
 
 
 
 
r   