
    Αi                        S SK r S SKrS SKrS SKJr  S SKJr  S SKJrJ	r	  S SK
rS SKrS SKJr  SSKJr  SSKJrJr  SS	KJrJr  SS
KJr  SSKJrJr  SSKJr  SSKJrJ r J!r!  \RD                  " S5        \RD                  " S5        \R                  RE                  S5         " S S5      r# " S S5      r$ " S S5      r% " S S\%5      r& " S S5      r'g)    N)OrderedDict)reduce)chainproduct)auto   )estimate_cost)OperatorDistAttrTensorDistAttr)DistributedContextDistributedOperatorContext)DistributedOperator)'get_distributed_operator_impl_containeris_elementwise_op)get_process_group) get_all_distributed_main_program+update_op_dims_mapping_by_default_dist_impl4update_op_dims_mapping_by_elementwise_like_dist_impl{   c                   D    \ rS rSr\S 5       r\S 5       r\S 5       rSrg)
PlanFilter/   c                     Sn[        U5      [        U5      :X  d   e[        U5       HD  u  pEUS:w  a$  X   X   -  S:w  d  UR                  U5      S:  a  SnUS:w  d  M7  U S   S:X  d  MB  SnMF     U$ )NTr   r   F)len	enumeratecount)process_mesh_topologytensor_shapedims_mappingvalididxdim_mappings         o/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/auto_parallel/static/planner.pycheck_dims_mapping_for_tensor(PlanFilter.check_dims_mapping_for_tensor0   s     < C$5555 ), 7Cb  %(=(JJaO#))+6:!Eb %:1%=%B !8     c                    UR                   nUc   S5       eU R                   H  nUR                  U5      n[        R	                  UR
                  X$   R
                  U5      (       d    gX$   R                  (       d  M]  [        U5      S:  d  Mn  USS   H  nUS:w  d  M      g   M     U R                   HH  nUR                  U5      n[        R	                  UR
                  X$   R
                  U5      (       a  MH    g   g)Nz$The process mesh should not be None.Fr   r   T)
process_meshinput_arg_namesget_input_dims_mappingr   r%   shapeis_datar   output_arg_namesget_output_dims_mapping)opop_dist_attrvarsr)   var_namer    dims          r$   check_dims_mapping_for_op$PlanFilter.check_dims_mapping_for_opC   s    #00'O)OO'**H'>>xHL;;""DN$8$8,  ~%%%#l*;a*?'+Cby$ , + ++H'??IL;;""DN$8$8,   , r'   c                    U R                   S:X  d   U R                   S:X  d  U R                   S:X  ah  U R                   H$  nUR                  U5       H  nUS:w  d  M      g   M&     U R                   H$  nUR	                  U5       H  nUS:w  d  M      g   M&     U R                   S:X  a<  U R                   H,  nUS:X  d  M  UR                  U5       H  nUS:w  d  M      g   M.     g)	Nelementwise_add
layer_normsoftmax_with_cross_entropyr   Flookup_table_v2pos_embeddingsT)typer*   r+   r.   r/   )r0   r1   r2   nameitems        r$   !check_dims_mapping_for_special_op,PlanFilter.check_dims_mapping_for_special_op[   s     GG((ww,&ww66**(??EDrz$ F + ++(@@FDrz$ G , 77''**++ , C CD I2:#( !J +
 r'    N)	__name__
__module____qualname____firstlineno__staticmethodr%   r5   r@   __static_attributes__rB   r'   r$   r   r   /   s<     $  .  r'   r   c                   j    \ rS rSr/ SQr/ SQr\S 5       r\S 5       r\S 5       r	\ S
S j5       r
Srg	)	PlanSpacet   )create_py_readercreate_double_buffer_readerread)lod_tensor_blocking_queue_0create_py_reader_0double_buffer_0c           	      (   [        [        S[        U 5      5      5      nU[        U5      :X  aq  Sn[        U5       H3  u  pU	S:w  d  M  XX   X	   -  S:w  d  UR	                  U	5      S:  d  M1  SnM5     U(       a%  UR                  [        R                  " U5      5        g[        [        U5      5       H\  n
X   (       a  M  U
S:w  a  SX'   UR                  Xj   5        [        R                  U UUUS-   UU5        SX'   UR                  5         M^     g)zCEnumerate dims mapping of tensor by the given process_mesh_topologyr   Tr   r   FN)listranger   r   r   appendcopydeepcopyrJ   _enum_dims_mappingpop)r   visitedpathdepthresr   numsr!   r"   r?   is              r$   rX   PlanSpace._enum_dims_mapping|   s    
 E"c"789:C%%E&t_	2:$),A,GG1L::d+a/ % - 

4==./s4y!A::6!%GJDG$,,)AI  #

 "r'   c                    U S:  d   S5       e/ n[        SU S-   5       H  nX-  S:X  d  M  UR                  U5        M      / n[        [        U5      S-
  SS5       GH  n/ nUR                  X   5        U[        U5      S-
  :X  a'  UR                  [        R                  " U5      5        MR  SnU[        U5      :  d  Me  [        U5      S:X  a  UR                  X   5        O[        U5      S:X  a  XS   US   -  -  S:X  a  XS   US   -  -  S:X  a'  UR                  [        R                  " U5      5        M  UR                  XS   US   -  -  5        UR                  [        R                  " U5      5        UR                  S5        UR                  S5        US-  nO)US   US   -  U :  a  UR                  S5        US-  nOGMr  U[        U5      :  a  GM  GM     U$ )z6Enumerate all process meshes with the given processes.r   z0The processes must be number and greater than 0.r   r      )rT   rU   r   rV   rW   rY   )	processesdivisorsr_   resultsresultjs         r$   enum_process_mesh_topology$PlanSpace.enum_process_mesh_topology   s    A~ 	
>	
~ q)a-(A}!" )
 s8}q(!R0AFMM(+&CMA%%t}}V45Ac(m#v;!#MM(+.[A% 1Iq	$9:a?$VAY)>?1D#NN4==+@A!"MM)q	F1I8M*NO#NN4==+@A"JJrN"JJrNFA!!9vay09<"JJrNFA!' c(m## 18 r'   c                 
   U R                  5       R                  n[        5       n/ n[        UR                  5      n[        UR                  UR                  5       H  n[        [        [        [        S[        UR                  5      5      5      5      5       Vs/ s H  nSPM     n	nSn
/ n/ n[        R                  UR                  U	UU
UX7   R                  5        [        R                  " U5      XG'   M     [        [!        UR#                  5        Vs/ s H  oU   PM	     sn6 5      nU GH  n[%        5       nUUl        [        UR#                  5       5      n[)        U5       Hq  u  nnUU   UR                  ;   a  UR+                  UU   U5        M0  UU   UR                  ;   a  UR-                  UU   U5        MZ  [/        SR1                  SSS95      e   [3        UU5      nUGcn  [5        UR                  5      (       a  S	nS	n [7        U5      nU(       a  U(       d  [:        R=                  UUR>                  U5      (       ac  [:        RA                  UUR>                  U5      (       a=  S
UR>                  l!        SUR>                  l"        URG                  UR>                  5        GM  S	nS	n [I        U5      nU(       a  U(       d  [:        R=                  UUR>                  U5      (       ac  [:        RA                  UUR>                  U5      (       a=  SUR>                  l!        SUR>                  l"        URG                  UR>                  5        GM,  URJ                  n[)        U5       H  u  nnURM                  U5      (       d  M  [:        R=                  UUR>                  U5      (       d  MF  URN                  R                  UR>                  l!        UUR>                  l"        URG                  UR>                  5        M     GM     U(       d  [%        5       nUUl        UR                   H>  nUR+                  X7   RP                  X7   R                   Vs/ s H  nSPM     sn5        M@     UR                   H>  nUR-                  X7   RP                  X7   R                   Vs/ s H  nSPM     sn5        M@     [3        UU5      nSUR>                  l!        SUR>                  l"        URG                  UR>                  5        U$ s  snf s  snf ! [8         a  nSn SnAGNSnAff = f! [8         a  nSn SnAGNSnAff = fs  snf s  snf )zQEnumerate the valid distributed attribute for op based on the given process mesh.r   Fr   z0The {varname} is not input or output of op {op}.zvar_names[idx]r0   )varnamer0   NTelementwisedefault))global_blockr2   r   r   r=   r   r*   r.   rT   r   rS   r,   rJ   rX   rV   rW   r   keysr
   r)   r   set_input_dims_mappingset_output_dims_mapping
ValueErrorformatr   r   r   	Exceptionr   r5   	dist_attrr@   	impl_typeimpl_idxrU   r   implsis_auto_compatible	serial_opr>   )programr0   r)   r2   dims_mapping_dictop_valid_dist_attrsdist_op_impl_containerr3   _rZ   r\   r[   dims_mapping_listkeycomposed_dims_mapping_listcomposed_dims_mappingr1   	var_namesr"   r    dist_opchangedr!   erx   implr_   s                              r$   _enum_valid_dist_attr_for_op&PlanSpace._enum_valid_dist_attr_for_op   s    ##%**'M !HGG"

 b00"2E2EFH s4b#l6H6H2I(J#KLMMA M   ED "((""!$$ +/--8I*J'! G& &*4E4J4J4LM4LSC(4LM&
"
 &@!+-L(4L%.3356I%./D%E!\S>R%7%77 77!# s^r':':: 88!# %JQQ$4 R   &F  *"l;G%-$RWW--"G E&"V##
 W%?? 1 14 (JJ 1 14  ;HG--79:G--6/66w7H7HI"G E&"M##
 W%?? 1 14 (JJ 1 14  ;DG--79:G--6/66w7H7HI*00E&u-	T**733!;;G--t  7>6G6G6L6L))358))2+2273D3DE . &@R #+-L(4L%..33N''dn6J6J)K6J"6J)K / //44N''dn6J6J)K6J"6J)K 0
 *"l;G*3G')*G&&&w'8'89""c( NB % & %&& % & %&: *L *LsH   TTT6T5!U/U
T2%T--T25
U?UUc                    [        5       nU R                  5       R                  nU R                  5       R                  n[	        S US5      n[        [        U5      5      nSnSn	U(       a  US   n
[        U5      U
-  n[        U5      S:  as  USS nXj-  n[        U
5       Vs/ s HS  n[        R                  " [        R                  " UX-  US-   U-   5      R                  U5      R                  5       S9PMU     n	nO[        U5      S:X  a.  [        U
5       Vs/ s H  n[        R                  " U/S9PM     n	nOi[        U5      S:  aF  [        R                  " [        R                  " U5      R                  U5      R                  5       S9nO[        R                  " US9n[        U5       GH  u  nnSnUnSnU	b?  UW-  [        U	5      :  a  X-  OX-  S-
  nU[        U	5      :  a  [        U	5      S-
  nU	U   nUR                  [         R"                  ;   a  [%        5       nUUl        UR(                   H\  nU[         R*                  ;   a  UR-                  U/ 5        M+  UU   R.                   Vs/ s H  nSPM     nnUR-                  UU5        M^     UR0                   H\  nU[         R*                  ;   a  UR3                  U/ 5        M+  UU   R.                   Vs/ s H  nSPM     nnUR3                  UU5        M^     U/nUS:w  a  SOUnO[         R5                  U UU5      nUc   SU S35       eUU/UUR6                  R9                  5       '   GM     UU	U4$ s  snf s  snf s  snf s  snf )	z>Enumerate valid distributed attributes for all ops in program.c                 
    X-  $ NrB   )xys     r$   <lambda><PlanSpace.enum_valid_dist_attr_for_program.<locals>.<lambda>S  s    r'   r   Nr   )meshr   z
Enumerate z$ valid distributed attribute failed.)r   rn   opsr2   r   rS   rT   r   r   ProcessMeshnparrayreshapetolistr   r=   rJ   not_enum_opsr
   r)   r*   special_varsrp   r,   r.   rq   r   descid)r{   r   is_pipelinevalid_dist_attr_dictr   r2   rc   global_groupglobal_process_meshpipeline_process_meshespipeline_stagesop_count_per_stageprocess_mesh_shapeper_process_mesh_groupr_   r"   r0   r}   op_process_meshpipeline_stager1   r3   r    s                          r$    enum_valid_dist_attr_for_program*PlanSpace.enum_valid_dist_attr_for_programJ  s   
  +}""$((##%**-/DaH	E),-""& 3B7O!$S_!<()A-%:3B%?")2)E& #?3+ 4 $$XX( ! :a!e"8>9 !!34	 4 ( +' *+q08=o8N+8N1D$$1#.8N ( + ()A-&*&6&6,/W23VX'# '+&6&6L&I# !~GC"&1ON&2 0037N3OO -2Q6 
 "S)@%AA%()@%AA%EN"9."Iww)000/1,;) " 2 2H9#9#99$;;HbI48N4H4H'I4Hq4H'I$;;$l !3 !# 3 3H9#9#99$<<XrJ48N4H4H'I4Hq4H'I$<<$l !4 (4n#&4&:&/&L&LR'# '2 RD DE2 $2 .Y &d !#
 	
[++F (J (Js   $AM%M*9M/%M4rB   N)F)rC   rD   rE   rF   r   r   rG   rX   rh   r   r   rH   rB   r'   r$   rJ   rJ   t   sh    NLL    D ) )V |# |#| 49e
 e
r'   rJ   c                   0    \ rS rSrS r\S 5       rS rSrg)SearchAlgorithmi  c                     Xl         g r   )_name)selfr>   s     r$   __init__SearchAlgorithm.__init__  s    
r'   c                 &    U R                   U l        g r   r   r>   r   s    r$   r>   SearchAlgorithm.name  s    JJ	r'   c                     [        S5      e)Nz)Please Implement this method in subclass.)NotImplementedErrorr   s    r$   searchSearchAlgorithm.search  s    !"MNNr'   r   N)	rC   rD   rE   rF   r   propertyr>   r   rH   rB   r'   r$   r   r     s!      Or'   r   c                      ^  \ rS rSrSU 4S jjr\S 5       r\S 5       r\S 5       rS r	S r
 SS jrS	 rS
 r SS jr SS jrS rSrU =r$ )MCMCi  c                 H   > [         TU ]  S5        Xl        X0l        X l        g )Nmcmc)superr   _serial_program_info_max_search_times_parallelizer)r   serial_program_infoparallelizermax_search_times	__class__s       r$   r   MCMC.__init__  s"     $7!!1)r'   c                     U R                   $ r   r   r   s    r$   r   MCMC.serial_program_info      (((r'   c                     U R                   $ r   r   r   s    r$   r   MCMC.parallelizer      !!!r'   c                     U R                   $ r   )r   r   s    r$   r   MCMC.max_search_times      %%%r'   c                    UR                   S:X  Ga9  UR                   GH'  nUR                  U5      R                  U5      nUUR	                  X6   5      R
                  :w  d  MG  SnU H  n	XiR                  ;   a  UU	R                  R                  5          S   n
U
 H  nUR                  U5      U:X  d  M  UR                  X5        U	R                   HH  n[        5       nUR                  Ul        UR                  U5      Ul        UR                  X<   U5        MJ     Sn  O   U(       d  M    O   U(       a  GM  [        S5      e   g g )Nr:   Fr   Tz2Change softmax_with_cross_entropy dist attr failed)r=   r*   get_op_dist_attr_for_programr+    get_tensor_dist_attr_for_programr    r.   r   r   r/   set_op_dist_attr_for_programr   r)    set_tensor_dist_attr_for_programrr   )r   r0   r   r2   dist_contextr   r3   r    has_changed	search_opop_dist_attr_listr1   r>   tensor_dist_attrs                 r$   make_special_op_unshardMCMC.make_special_op_unshard  sr    7722..+HH ((2  !#DD"l#
 #(K%(	#'A'AA0D ) 1 1 311!- 1B$0$H$H(0%& (4%4
 %1$M$M(1%& 1:0J0J;I;K(8,8,E,E )9(E IUHlHl,0I*(8(E )5(U(U,0J8H)* 1K 37K$)- 1B. ';!; &)< ';(P U / 3r'   c                 z   UR                  5       R                  nUR                  5       R                  n[        5       nU GHB  nXR                  R                  5          S   n	[        R                  R                  [        U	5      5      n
X   nUR                  X5        UR                   Hg  nUS:X  a  M  UR                  Xl   5       b  M"  [        5       nUR                  Ul        UR                  U5      Ul        UR#                  Xl   U5        Mi     UR$                   HH  n[        5       nUR                  Ul        UR'                  U5      Ul        UR#                  Xl   U5        MJ     U R)                  XXgU5        GME     Ub  UR+                  U5        U$ Ub  U H  nUR+                  U5        M     U$ )Nr   rO   )rn   r   r2   r   r   r   r   randomrandintr   r   r*   r   r   r)   r+   r    r   r.   r/   r   add_process_mesh)r   r   r{   r   r   r   r2   new_dist_contextr0   op_valid_dist_attr_listrandom_op_dist_attrinit_op_dist_attrr3   r   r)   s                  r$   init_programMCMC.init_program  s    ""$((##%**-/B&:77::<&H&K#"$))"3"3+,# !8 L99"P..<<$EE 
 (6'7$)66 %1 *@@J %1 %EE(8! /( //#1#3 0A0N0N -%==hG !- !AAN$4 0 ((1EM V *--.AB
  	 %0 7 11,? !8  r'   c                    S n[        U R                  XR                  5      nUb  U Vs/ s H  nUR                  PM     snOS nSnU H[  nSn	UR	                  5        H9  n
U
R
                  (       d  M  SU
R                  ;   d  M(  U
R                  S   nSn	  O   U	(       d  M[    O   SSKJ	n  U" U5      n[        US UUUS9nU$ s  snf )Nr   Fz@RESHARDr   T)get_standalone_cost_data)clusterpipeline_configstandalone_cost_data
batch_size)r   r   r   process_ids	list_varsr-   r>   r,   utilsr   r	   )r   r   r   costall_dist_main_programr)   r   microbatch_sizer{   searched_batch_sizevarr   r   s                r$   estimate_searched_strategy_cost$MCMC.estimate_searched_strategy_costB  s      @$$l4E4E!
 '2 %<$;L (($;
  	 ,G"'((*;;;:#9&)iilO*.'	 +
 #" - 	478MN !+!5&
 ?s   Cc                    UR                    HI  nUR                  n[        5       nXgl        UR                  U5      Ul        UR                  X5   U5        MK     UR                   Hq  nX5   R                  (       d  X5   R                  (       d  M+  UR                  n[        5       nXgl        UR                  U5      Ul        UR                  X5   U5        Ms     g r   )
r.   r)   r   r/   r    r   r*   is_parameterr-   r+   )r   r0   r1   r2   r   r3   r)   r   s           r$   set_tensor_dist_attrMCMC.set_tensor_dist_attrl  s    ++H'44L-/,8)44X> ) 99 0 , **H~**dn.D.D.D+88#1#3 0<- 77A !- ==N$4 +r'   c                 ,   UUR                  U5      l        UR                   H  nUUR                  X5   5      l        M     UR                   HC  nX5   R
                  (       d  X5   R                  (       d  M+  UUR                  X5   5      l        ME     g r   )r   r)   r.   r   r*   r   r-   )r   r0   changed_process_meshr2   r   r3   s         r$   change_process_meshMCMC.change_process_mesh  s     . 	11	

++H 2 99 , **H~**dn.D.D.D "6 ==N	 +r'   c                    UR                  5       R                  n/ nU H4  nUR                  [        R                  ;  d  M#  UR                  U5        M6     U(       d   S5       eUR                  5       R                  n[        R                  " U5      n	[        5       U	l
        S n
[        R                  R                  [        U5      5      nXk   nX,R                  R!                  5          S   nX,R                  R!                  5          S   n[        R                  R                  [        U5      5      n[        R                  " X   5      nUS   R                  R!                  5       nUS:  Ga  [        R                  " U5      n
[        R                  R                  S5      nUS:X  a'  U	R#                  UU5        U R%                  UUX5        GOUS:X  GaT  US-
  nUS:X  dS  U[        U5      S-
  :X  dA  US-   [        U5      S-
  :X  aT  U
XkS-      R                  R!                  5          S   US-   :X  a(  U	R#                  UU5        U R%                  UUUU	5        GO,UU   nXkS-      R                  R!                  5       nU
U   S   US-   :X  aK  US-   [        U5      S-
  :w  a6  XU   S'   U
U   S    H
  nUUl        M     U R)                  XkS-      UUU	5        UXR                  R!                  5          S'   UU   nUUl        U
UR                  R!                  5          S    H
  nUUl        M     U	R#                  UU5        U R%                  UUUU	5        [+        US-
  SS5       H  nXU   R                  R!                  5          S   nU
UU   R                  R!                  5          S   nUU   nUUS-   :X  ad  UXU   R                  R!                  5          S'   U H
  nUUl        M     UU	R-                  UU   5      l        U R)                  UU   UUU	5        M    GOb   GO^US-   nU[        U5      :X  d8  US:X  d2  U
XkS-
     R                  R!                  5          S   US-
  :X  a.  US:X  a(  U	R#                  UU5        U R%                  UUUU	5        GOUU   nXkS-
     R                  R!                  5       nU
U   S   US-
  :X  a<  US:w  a6  XU   S'   U
U   S    H
  nUUl        M     U R)                  XkS-
     UUU	5        UXR                  R!                  5          S'   UU   nUUl        U
UR                  R!                  5          S    H
  nUUl        M     U	R#                  UU5        U R%                  UUUU	5        [+        US-   [        U5      5       H  nXU   R                  R!                  5          S   nU
UU   R                  R!                  5          S   nUU   nUUS-
  :X  ad  UXU   R                  R!                  5          S'   U H
  nUUl        M     UU	R-                  UU   5      l        U R)                  UU   UUU	5        M    O(   O%U	R#                  UU5        U R%                  UUX5        U H(  nUR                  S:X  d  M  U R/                  XvXU5          O   U
c  X)4$ X4$ )Nz2The ops of program have no distributed attributes.r   r   r      r:   )rn   r   r=   rJ   r   rU   r2   rV   rW   r   _dist_op_contextr   r   r   r   r   r   r   r   r)   r   rT   r   r   )r   r{   r   r   r   raw_opsr   r0   r2   r   new_valid_dist_attr_dictrandom_selected_op_idxselected_opr   r   random_selected_dist_attr_idxselected_op_dist_attr	start_idxchanged_modechanged_stageselected_op_process_mesh
next_op_idr1   new_process_meshr"   stagevalid_dist_attr_list	pre_op_ids                               r$   search_onceMCMC.search_once  s    &&(,,Bwwi444

2  HHHs##%**==6,F,H)#' !#!2!23s8!<1"67G7G7J7J7L"Ma"P-.>.>.A.A.CDQG(*		(9(9'()
% !%#B!
 FKKNN$	B (,}}5I'J$99,,Q/Lq  ==!6 ))!6 " . 2!R'-SA=.2c#hlB4 :;@@CCE *A-. %AA#%: --#-(	 0G&0, "%a%?!@!E!E!H!H!JJ0<Q?)A-.2Q6#c(Q,FBP<Q?,D&--L 9QL5-
 00 :;4 ,	 & --=-=-@-@-BCAF (?}'M$9I)6(@#((++-)) 5E1) %AA#%: --#-(	  %%;a%?RH 8S9I9I9K LQ O/GHMM,,.00, ,C),( !MA$55 - 5X]]5E5E5GHK 1E<L 9 1E .> -II #C* 44 #C 0 $ 0	 "3  I8 !/ 2!S)@%AA-20 :;@@CCE *A-. 4q8 %AA#%: --#-(	 0G&0, !$Q$> ? D D G G II0;A>)A-.2a7AO;A>,DY,O-L 9QL5-
 00 :;4 ,	 & --=-=-@-@-BCAF (?}'M$9I)6(@#((++-)) 5E1) %AA#%: --#-(	  %%;a%?SJ 8S9I9I9K LQ O/GHMM,,.00, ,C),( !MA$55 - 5X]]5E5E5GHK 1E<L 9 1E
 .> -II #C* 44 #C 0 $ 0	 "5  K8 992 %%2D Bww66,,T5I   $+'99+==r'   c                 d   SnUnU R                  X#5      R                  nUnX@R                  :  a~  US-  nU R                  U R                  R
                  UUU5      S   nU R                  X5      R                  n	Xy-
  S:  a  [        R                  " U5      nU	nSnX@R                  :  a  M~  XW4$ )Nr   r   )r   runtimer   r  r   train_programrV   rW   )
r   r   init_dist_contextr   timesbest_dist_contextr   min_costr   cur_costs
             r$   _search_coreMCMC._search_core  s     -33

' 	 +++QJE#//((66$!'	 
   ;; g  #q($(MM2B$C!# +++ !**r'   c                    [        S5        [        R                  " 5       nU R                  R                  nU R                  R                  nUc  [
        R                  R                  5       O[        UR                  S5      5      nUS:  d   S5       e[        R                  U5      nS nS nS nS n	U H  n
[        SU
 S35        [        R                  X*S5      u  nnnU R                  UUUU5      nU R                  XU5      u  nn[        SU S	U
 S35        [        5       Ul        U	c  UOU	n	Uc  UOUnU	U:  d  M  UnUn	M     S nS nU H  n
[        U
5      S
:X  a  M  [        SU
 S35        [        R                  X*S5      u  nnnU R                  UUUU5      nU R                  XU5      u  nn[        SU S	U
 S35        [        5       Ul        Uc  UOUnUc  UOUnUU:  d  M  UnUnM     UU	:  a  UnU	n[        S5        OUnUn[#        S5      nUR$                   H  nUR'                  UR(                  5        M      [        R                  " 5       n[        SU SUU-
   S35        Xg4$ )NzStart MCMC searching.GPUr   zGet process failed.z!MCMC search: search process mesh z with pipeline mode.TzMCMC search: the min cost is z in the process mesh r  z without pipeline mode.FzHBetter set FLAGS_benchmark=1 to avoid hang problem in the pipeline mode.z$End MCMC searching: the min cost is z and the search time is zs.)printtimer   r  r   paddledistributedget_world_sizer   get_all_devicesrJ   rh   r   r   r  r   r  r   _process_meshes	add_ranksr   )r   
start_timer  r   rc   process_mesh_topology_listsearched_dist_contextr  searched_pipeline_dist_contextpipeline_min_costr   r   r   r   r  r  r   "searched_non_pipeline_dist_contextnon_pipeline_min_costpg0r)   end_times                         r$   r   MCMC.search  sA   %&YY[
00>>**22  --/W,,U34 	
 1}333}%.%I%I&
" !%)-& %?!34I3JJ^_ ::d	$'# !% 1 1$'#	! '+&7&7$9P'#t /v5JK`Jaauv 2L1M.)17H 
 29 "3 +
 !4'1B.$(!E &@H .2* $%?!()Q.34I3JJab ::e	$'# !% 1 1$'#	! '+&7&7$9P'#t /v5JK`Jaaxy 2L1M.-5;P "
 6= "7 /
 %t+5F2(,%K &@N !#44$B!(HZ %G!,H  "1AALMM,223 B99;28*<TU]`jUjTkkmn	
 %..r'   )r   r   r   )   r   )rC   rD   rE   rF   r   r   r   r   r   r   r   r   r   r   r  r  r   rH   __classcell__)r   s   @r$   r   r     s    * ) ) " " & &0d< ~ 59(T46& !%{>B !%	+:s/ s/r'   r   c                   l    \ rS rSr SS jr\S 5       r\S 5       r\S 5       r\S 5       r	S r
S	 rS
rg)Planneri$  Nc                 T    Xl         X l        X0l        U R                  U5      U l        g r   )r   r   _algorithm_configcreate_algorithm_searcher_algorithm_searcher)r   r   r   algorithm_configs       r$   r   Planner.__init__%  s,     %8!)!1#'#A#A$
 r'   c                     U R                   $ r   r   r   s    r$   r   Planner.serial_program_info/  r   r'   c                     U R                   $ r   )r7  r   s    r$   r:  Planner.algorithm_config3  r   r'   c                     U R                   $ r   )r9  r   s    r$   algorithm_searcherPlanner.algorithm_searcher7  s    '''r'   c                     U R                   $ r   r   r   s    r$   r   Planner.parallelizer;  r   r'   c                    UR                  SS 5      nUc   S5       eS nUS:X  aZ  UR                  SS 5      nUb#  [        U R                  U R                  U5      nU$ [        U R                  U R                  5      nU$ [	        S5      e)Nr>   zInvalid algorithm config.r   r   z4Other search algorithms have not been supported now.)getr   r   r   r   )r   r:  r>   rA  r   s        r$   r8  !Planner.create_algorithm_searcher?  s    ##FD1<!<<!6>/334FM $/ ,,%%$  "! $22D4E4EF  "!	 &F r'   c                 6    U R                   R                  5       $ r   )rA  r   r   s    r$   r   Planner.searchW  s    &&--//r'   )r7  r9  r   r   r   )rC   rD   rE   rF   r   r   r   r:  rA  r   r8  r   rH   rB   r'   r$   r5  r5  $  sd    BF
 ) ) & & ( ( " ""00r'   r5  )(rV   r   r!  collectionsr   	functoolsr   	itertoolsr   r   numpyr   r"  paddle.distributed.fleetr   
cost_modelr	   dist_attributer
   r   r   r   r   r   r   operators.commonr   r   process_groupr   r   r   r   r   seedr   rJ   r   r   r5  rB   r'   r$   <module>rT     s       #  $   ) % < H ( -  C  C  		s B BJ|
 |
~		O 	Ob	/? b	/J40 40r'   