
    Αiw                        S SK Jr  S SKrS SKJr  S SKJrJr   S SKJr  SSK
Jr  SS	KJrJrJrJr  SS
KJr  SSKJr  SSKJrJr  / rS r " S S\5      rg! \	 a    Sr NAf = f)    )annotationsN)	framework)P2POpbatch_isend_irecv)deep_ep   )logger   )FakeMicroDatasetHybridParallelOptimizerPipelineDatasetPreprocessorPipelineParallel)BatchCommHelper)ScheduleChunk)
EventStoreWeightGradStorec                4    U R                  5       nSUl        U$ )NF)detachstop_gradient)xos     p/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/fleet/meta_parallel/dualpipev.pydetach_and_requires_gradr   /   s    	
AAOH    c                    ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
 rS!S"S jjrS rS!S jrS!S jrS#S$S jjr   S%     S&S jjr S' S(S jjrS(S jrS(S jrS"S jrS"S jrS"S jrS"S jr   S)       S*S jjr   S+         S,S jjr      S-       S.S jjrS rS rS r  S/S jr   S0S jr!S r"U =r#$ )1DualPipeVParallel5   zv
An implementation of the DualPipeV, based on
https://github.com/deepseek-ai/DualPipe/blob/main/dualpipe/dualpipe.py.
c                  > [         TU ]  XUS9  [        [        U R                  5      S5      U l        [        R                  " SU R
                   35        U R                  U l	        U R                  R                  U l        U R                  R                  U R                  S-
  U R                  R                  -     U l        U R                  R                  U R                  S-   U R                  R                  -     U l        U R#                  5       U l        U R#                  5       (       + U l        [)        U R*                  5      U l        g )N)layershcgstrategyoverlapped_forward_backwardz:Using DualPipeVParallel with overlapping forward backward=r
   )super__init__hasattrtype_layersr"   r	   info
num_stages	num_rankspp_grouprank
group_rankranks
world_size	prev_rank	next_rankis_pipeline_first_stageneed_broadcast_metaneed_recv_metar   _using_cache_p2p_helper)selfr   r    r!   	__class__s       r   r$   DualPipeVParallel.__init__;   s   (C+2 =,
( 	HIiIiHjk	
 --,,,,__q DMM$<$<<
 ,,__q DMM$<$<<
 $(#?#?#A "&">">"@@*4+<+<=r   c                     U R                   S:H  $ Nr   )r-   r7   s    r   r2   )DualPipeVParallel.is_pipeline_first_stageS   s    !##r   c                :    U R                   U R                  S-
  :H  $ Nr
   )r-   r*   r<   s    r   is_pipeline_last_stage(DualPipeVParallel.is_pipeline_last_stageV   s    $..1"444r   c                   / / 4U l         / / 4U l        / / 4U l        / / 4U l        / U l        / / 4U l        / U l        SS/U l        SS/U l        SS/U l	        SS/U l
        SS/U l        SS/U l        / U l        / U l        / U l        g r;   )input_tensorsoutput_tensorsinput_grad_tensorsoutput_grad_tensorsloss_tensorsschedule_chunksloss_fn_chunkscurrent_f_acc_idcurrent_b_acc_idcurrent_send_f_acc_idcurrent_send_b_acc_idcurrent_recv_f_acc_idcurrent_recv_b_acc_idcomm_forward_opscomm_backward_opsto_freer<   s    r   _reset_statesDualPipeVParallel._reset_statesY   s     "X!2h#%r($&8 13 "Bx  "#A!"A&'V"&'V"&'V"&'V"-/.0,.r   c                   U R                  5       =(       a    US:H  nU(       a1  Uc   eU R                  U   R                  [        X   5      S   5        U R                  (       a  S U R                  U   U'   U R                  U   U   $ r;   )r2   rC   appendnextforward_only)r7   micro_datasetsphaseacc_idis_first_stages        r   _get_forward_inputs%DualPipeVParallel._get_forward_inputsm   s    557FEQJ!---u%,,T.2G-H-KL04Du%f-!!%(00r   c                    U R                  5       =(       a    US:H  nU(       a9  U R                  (       a(  Uc   e[        X   5      S   nU R                  U5        U$ g r?   )r2   _compute_lossrW   _check_micro_batch_data_valid)r7   rY   rZ   r[   is_last_stagelabelss         r   _get_forward_labels%DualPipeVParallel._get_forward_labelsv   sX    446E5A:T//!---./03F..v6Mr   c                ^   U R                  XU5      nS nU R                  (       du  U R                  R                  S   " XE5      n[        R
                  R                  SS9   U R                  S:  a  U R                  (       d  XpR                  -  nS S S 5        OpU R                  R                  S   R                  5       nXVl
        U R                  S:  a"  U R                  (       d  U R                  Ul        UR                  U5      nU R                  X'U5        g ! , (       d  f       N!= f)Nr   Fenabler
   )rd   r"   r'   _loss_fnpaddleamp	auto_castaccumulate_steps_delay_scale_lossbuild_schedule_noderc   scale_loss_factorforward_store_forward_loss)r7   rY   rZ   r[   logitsrc   loss_fn_nodeloss_tensors           r   _loss_computeDualPipeVParallel._loss_compute   s    )).H//,,//26BK%%U%3((1,T5K5K"-0E0E"EK 43  <<003GGIL"($$q(1G1G151F1F.&..v6K  \B 43s   "0D
D,c                t   U R                   U   R                  U5        U R                  5       (       a>  US:X  a8  U R                  S   R                  U Vs/ s H  n[	        U5      PM     sn5        U R                  5       =(       a    US:H  nU(       d  U R                  U   R                  U5        g g s  snf Nr   r
   )rH   rV   r@   rC   r   r2   rD   )r7   rZ   outputsschedule_chunkoutputrb   s         r   _store_forward_tensors(DualPipeVParallel._store_forward_tensors   s    U#**>:&&((UaZq!((@GHf)&1H 446E5A:&--g6  Is   B5c                   U R                   U   nU R                   U==   S-  ss'   U R                  X!U5      nU R                  (       a+  U R                  R	                  US9nUR                  U5      nOS nU R                  R                  XAS9n[        U[        R                  5      (       a  U/OUnU R                  5       =(       a    US:H  nU(       a#  U R                  (       a  U R                  X!X65        U R                  XU5        g )Nr
   chunk_id)rJ   r]   r"   r'   get_schedule_chunkrq   
isinstancerj   Tensorr2   r`   rv   r}   )r7   rZ   rY   r[   inputsr{   rz   rb   s           r   _forward_compute"DualPipeVParallel._forward_compute   s    &&u-e$)$)).H++!\\<<e<LN$,,V4G!Nll**6*BG)'6==AA7)w446E5A:T//~fF##ENCr   c                   U R                   U   U   nS U R                   U   U'   U R                  U   U   nS U R                  U   U'   [        X45       VVs/ s H  u  pVUc  M
  XV4PM     nnn[        [        U6 5      u  p4X44$ s  snnf N)rD   rF   ziplist)r7   rZ   r[   rz   output_gradstg	non_emptys           r   _get_backward_inputs&DualPipeVParallel._get_backward_inputs   s    %%e,V4-1E"6*//6v>26  '/"79
9tqQFQF9 	 
 !%S)_ 5$$	
s   	B$Bc                   UcF  U R                   U   U   nU Vs/ s H'  nUc  M  UR                  (       a  M  UR                  PM)     nnS U R                   U   U'   [        U[        R
                  5      (       a  U4nU R                  5       (       a%  US:X  a  U R                  S   R                  U5        g U R                  U   R                  U5        g s  snf )Nr
   r   )
rC   r   gradr   rj   r   r@   rF   rV   rE   )r7   rZ   r[   input_gradsr   r   s         r   _store_backward_tensors)DualPipeVParallel._store_backward_tensors   s    ''.v6F  A *+//   
 -15!&)k6==11&.K&&((UaZ$$Q'..{;##E*11+>s   CCCc                   U R                  5       =(       a    US:H  nU(       a  U R                  (       a  [        U[        [        45      (       a  [        U5      S:X  d   eUS   n[        U[        R                  5      (       d   S5       eU R                  R                  U5        U R                  R                  U5        g g g )Nr
   r   z4Currently, loss_fn should obtain Paddle.Tensor dtype)r2   r`   r   tupler   lenrj   r   rG   rV   rI   )r7   rZ   ru   rt   rb   s        r   rr   %DualPipeVParallel._store_forward_loss   s    446E5A:T//+t}55;'1,,,)!nk6==99 F9 $$[1&&|4 0=r   c                T   U R                   (       a  g U R                  U   nU R                  U==   S-  ss'   U R                  5       =(       a    US:H  nU[        l        S n[
        R                  R                  SS9   U(       Ga  U R                  U   nU R                  (       a{  U R                  U   nU R                  U   U   nU R                  R                  [        / 5      S S UUS U R                  S S S9	u    pS U R                  U'   S U R                  U   U'   GO1U R                  (       a9  [
        R                  R!                  U R                  R#                  U5      5        O[
        R                  R!                  U5        OU R%                  X5      u  pU R                  (       aR  U R                  U   U   nU R                  R                  [        / 5      S S US US S S S9	u    pS U R                  U   U'   OQ['        U
5      S:  aB  U
 Vs/ s H  oR(                  (       a  M  UPM     n
n[
        R                  R!                  U
US9  S S S 5        S[        l        U(       a  [        R*                  " 5         U R-                  XUS9  g s  snf ! , (       d  f       NK= f)Nr
   Frg   combine_bw_event_to_wait	pp_streamr   )tensorsgrad_tensorsr   )rX   rK   r2   r   enabledrj   rk   rl   rG   r"   rI   rH   r'   r   scalerautogradbackwardscaler   r   r   flushr   )r7   rZ   	enable_zbr[   rb   r   lossrt   backward_chunk_rz   r   r   s                r   _backward_compute#DualPipeVParallel._backward_compute   sa   &&u-e$)$446E5A:"+ZZ!!!/((033#'#6#6v#>L%)%9%9%%@%HN@@)"-  *(  KK59&* A 
 &Aq 37D''/:>D((/7{{001B1B41HI006(,(A(A%(P%33%)%9%9%%@%HN@@)"-  * ( 59&* A 
 &Aq ;?D((/77|a'.5"Mg__1g"M00$+)5 1 _ 0f #(!!#$$U$L #N] 0/s%   ?FJJ.J4 JJ
J'c                   U R                   (       a  U R                  X5        g U R                  (       d#  U R                  X5        U R                  U5        g U R                  U   nU R                  U==   S-  ss'   U R                  X1U5      nU R                  X1U5      nUb`  U R                  R                  S   R                  5       n	Xl
        U R                  S:  a"  U R                  (       d  U R                  U	l        OS n	U R                  U   n
U R                  U==   S-  ss'   U R                  5       =(       a    US:H  nU(       a  U R                   U
   nS nOS nU R#                  X*5      u  pU R                  R%                  US9nU R&                  U   U
   nU R                  R                  UUU	UUUU R(                  UU(       aA  U R*                  R,                  R/                  [0        R2                  R5                  5       5      OS S9	u  nnnS U R&                  U   U
'   U R7                  UUU5        U R9                  UUU	5        U R;                  X*US9  g )Nr
   r   r   r   r   )rX   r   r"   r   rJ   r]   rd   r'   ri   ro   rc   rm   rn   rp   rK   r2   rI   r   r   rH   r   r+   process_group
get_streamrj   r   _current_expected_place_r}   rr   r   )r7   forward_phasebackward_phaserY   combine_backward_event_to_waitpass_pp_streamforward_acc_idforward_inputsforward_labelsforward_loss_fn_nodebackward_acc_idis_last_stage1backward_loss_fn_nodebackward_gradsr   forward_chunkr   forward_outputsforward_lossbackward_input_gradss                       r   _forward_backward_compute+DualPipeVParallel._forward_backward_compute  st    !!-@//!!-@"">2 ..}=m,1,11>
 11>
 %#'<<#8#8$!!# ! +9'$$q(1G1G9=9N9N$6#'  //?n-2-557ONa<O$($7$7$H!!N$(! $ 9 9!A 777O--n=oNLL44$%)G
 & MM//::((AAC  5  	<';& AE^,_= 	##?M	
 	  <)=	

 	$$9M 	% 	
r   c                   U R                   b  [        U R                   5      OSnU R                  b  [        U R                  5      OSnUS:X  ap  US:X  aj  [        R                  b  [        R                  nS [        l        U$ [
        R                  " [        R                  R                  5       R                  5      $ U=(       a    U R                  =(       a    [
        S LnU R                  R                  R                  [        R                  R!                  5       5      nU(       a%  U R                  R                  R#                  S5        US:  a5  [%        U R                   5      nU(       d  U H  n	U	R'                  5         M     U(       a%  U R                  R                  R#                  S5        U(       a  [
        R                  " U5      n
Sn[        R                  b  [        R                  R)                  [        R                  R+                  US95         [        R                  R-                  5         S S S 5        [        R.                  " S 5        U R                  R                  R#                  S5        SnUS:  a5  [%        U R                  5      nU(       d  U H  n	U	R'                  5         M     U(       a%  U R                  R                  R#                  S5        U(       a'  W
R-                  5         [
        R                  " U5      nO<[
        R                  " [        R                  R                  5       R                  5      n/ U l         / U l        U R1                  5         U$ ! , (       d  f       GN== f)Nr   TF)stream_base)rP   r   rQ   r   eventr   get_event_from_custom_streamrj   devicecurrent_streamr   _overlap_p2p_commr+   r   r   r   r   set_outer_waitr   waitstream_guardStreamcurrent_stream_waitset_free_tensors)r7   p2p_overlapuse_outer_event_waitcommon_forward_ops_numcommon_backward_ops_nume_tuse_stream_wait_eventpp_raw_streamfwd_reqsreqforward_event_to_waitbackward_outer_event_waitbwd_reqsr   s                 r   _commit_and_wait_comm'DualPipeVParallel._commit_and_wait_commt  s   
 $$0 %%& 	 %%1 &&' 	 
 "Q&+Ba+G+ &&#'
 
77,,.:: 
 JD22Jwd7J 	 33>>557
  MM''66t<!A%()>)>?H(#CHHJ $  MM''66u= $+$H$H%! %*!'++$$$?   446
 NN4 MM''66t<(,%"Q&()?)?@H(#CHHJ $ %MM''66u= !557'.'K'K($ (/'K'K,,.::($ !#!#''I s   !M66
Nc                r    U R                   (       a  g U R                  5         [        R                  " 5         g r   )rX   r   r   popr<   s    r   _weight_passDualPipeVParallel._weight_pass  s(    ""$ 	r   c                H    U R                  U R                  5        / U l        g r   )_release_outputrR   r<   s    r   r   DualPipeVParallel._free_tensors  s    T\\*r   c                   U R                  5       (       a  US:X  d  U R                  5       (       a  US:X  a  g U R                  U==   S-  ss'   U R                  R	                  U R
                  US:X  a  U R                  OU R                  U R                  U R                  S9nU R                  U   R                  U5        g )Nr   r
   alloc_on_comm_stream)r2   r@   rN   r6   append_irecvrP   r0   r1   r+   r   rC   rV   r7   rZ   r   s      r   _recv_forwardDualPipeVParallel._recv_forward  s    ((**uz''))eqj""5)Q.)""//!!#qjDNNdnnMM!%!7!7	 0 
 	5!((1r   c                   U R                  5       (       a  US:X  d  U R                  5       (       a  US:X  a  g U R                  U   nU R                  U==   S-  ss'   U R                  U   U   nU R                  R                  U R                  UUS:X  a  U R                  OU R                  U R                  U R                  5        SU l
        U R                  R                  U5        g )Nr
   r   F)r2   r@   rL   rD   r6   append_isendrP   r1   r0   r+   r3   rR   extendr7   rZ   r[   r   s       r   _send_forwardDualPipeVParallel._send_forward  s    ((**uz''))eqj++E2""5)Q.)%%e,V4%%!!#qjDNNdnnMM$$	
 $) G$r   c                   U R                   (       a  g U R                  5       (       a  US:X  d  U R                  5       (       a  US:X  a  g U R                  U==   S-  ss'   U R                  R                  U R                  US:X  a  U R                  OU R                  U R                  U R                  S9nU R                  U   R                  U5        g )Nr
   r   r   )rX   r2   r@   rO   r6   r   rQ   r1   r0   r+   r   rF   rV   r   s      r   _recv_backward DualPipeVParallel._recv_backward  s    ((**uz''))eqj""5)Q.)""//""#qjDNNdnnMM!%!7!7	 0 
 	  '..w7r   c                   U R                   (       a  g U R                  5       (       a  US:X  d  U R                  5       (       a  US:X  a  g U R                  U   nU R                  U==   S-  ss'   U R                  U   U   nS U R                  U   U'   U R
                  R                  U R                  UUS:X  a  U R                  OU R                  U R                  5        g ry   )rX   r2   r@   rM   rE   r6   r   rQ   r0   r1   r+   r   s       r   _send_backward DualPipeVParallel._send_backward  s    ((**uz''))eqj++E2""5)Q.)))%0815&v.%%""#qjDNNdnnMM		
r   c                    U(       a  U R                  U5        U R                  5         U R                  X5        U(       a  U R                  U5        g g r   )r   r   r   r   )r7   rZ   rY   recvsends        r   _forward_passDualPipeVParallel._forward_pass$  sE     u%""$e4u% r   c                    U(       a  U R                  U5        U R                  5         U R                  X5        U(       a  U R                  U5        g g r   )r   r   r   r   )r7   rZ   r   r   r   s        r   _backward_pass DualPipeVParallel._backward_pass4  sE     &""$u0& r   c	                *   U(       a  U R                  U5        U R                  U5        U R                  5       =(       a    US:H  (       + =(       d    U R                  5       =(       a    US:H  n	U R                  5       =(       a    US:H  (       + =(       d    U R                  5       =(       a    US:H  n
U=(       a;    U(       + =(       a-    U R                  =(       a    [
        S L=(       a    U	=(       a    U
nU=(       aI    U(       + =(       a;    U R                  =(       a(    [
        S L=(       a    U	=(       a    U
=(       a    U(       + nU R                  U(       + U5      nU R                  UUUUUS9  U R                  U5        U R                  U5        g )Nr
   r   )r   r   )
r   r   r2   r@   r   r   r   r   r   r   )r7   r   r   rY   recv0first_chunk
last_chunk
main_stagelast_stage_and_first_chunkneed_send_forwardneed_send_backwardr   r   combine_bw_wait_events                 r   _forward_backward_pass(DualPipeVParallel._forward_backward_passD  s    }-N+ ((*A}/A
 D))+B0B 	 ((*B~/B
 E))+C!0C 	
  ;;&&; t#; #9'9 	  11&&1 t#1 #9'9	1
 0/ 	 !% : :N0!
 	&&+@) 	' 	
 	=)N+r   c                D   [        U[        5      (       a  U" 5       n[        U[        5      (       d  [        U[        5      (       d  U$ [	        UU R                  5       =(       a    US:H  U R                  5       =(       a    US:H  U R                  U R                  5      nU$ )z_
for backward compatibility, wrap data to Fake FakeMicroDataset if it is of type list or tuple
r   r
   )r   r   r   r   r   r2   rm   micro_batch_size)r7   datarZ   micro_datasets       r   
_wrap_dataDualPipeVParallel._wrap_dataz  s     d7886D4''*T42H2HK(((*9uz((*9uz!!!!
 r   c                F   [        U[        5      (       d   S5       e[        R                  " 5       R                  (       d   S5       eU R                  5       (       a  Uc   S5       eOS nX l        X0l        U R                  R                  5         U R                  U5        U$ )Nz5optimizer should be HybridParallelOptimizer subclass.z*Please enable the generation of gradients.z7For the first and the last stage, the data must be set.)r   r   r   _dygraph_tracer	_has_gradr2   	optimizerlr_schedulerr'   train#register_sharding_comm_overlap_hook)r7   r  r  r  s       r   _prepare_training#DualPipeVParallel._prepare_training  s    )%<== 	
C	
= ((*44 	
8	
4 ''))# I# D"(00;r   c                   [         R                  " S/S5      nU R                  5       (       as  [        U R                  5      S:  d   S5       eU R                   H$  nXR                  5       R                  S5      -  nM&     U R                  (       a  XR                  -  n[         R                  R                  XR                  SS9  U$ )Nr
   float32r   z4train_batch() in last stage should obtain valid lossT)groupsync_op)rj   zerosr2   r   rG   r   astypern   rm   distributed
all_reducer+   )r7   loss_sum_tensorr   s      r   _broadcast_final_loss'DualPipeVParallel._broadcast_final_loss  s     ,,sI6''))t(()A- F- ));;=#7#7	#BB *%%#8#88%%==$ 	& 	
 r   c           
     V	   X l         U R                  nU R                  nU R                  S:  a  U R                  US-  :  d   SU R                  < SU< 35       eX0l        U R                  5         U R                  R                  U R                  U R                  5        SU l	        U R                  US5      nU R                  US5      nXx/n	Xe-
  S-
  S-  n
[        U
5       H  nU R                  SU	5        M     US-   nU R                  S5        [        U5       Hi  nU R                  SU	SSS9  U R                  S5        U R                  SU	U R                  5       (       + =(       d    XS-
  :  S9  U R                  S5        Mk     Xe-
  S-
  n[        U5       HE  nU R!                  SS	S
9  U R                  S5        U R#                  5         U R                  SU	SS9  MG     U R                  US-  -
  U-   S-   nXe-
  S-
  S:  n[        U5       H  nUS-   U:H  nUS:X  a  U R                  5       (       ak  U R                  SU	SSS9  U R                  S5        U R!                  SSS9  U R                  S5        U R%                  S5        U R'                  SSU	S	US	S9  M  U R'                  SSU	SS	S	S9  U R'                  SSU	US	S9  M  U R'                  SSU	S	U R                  5       S9  U R'                  SSU	US	S9  M     Xe-
  S-
  n[        U5       H'  nU R!                  S5        U R'                  SSU	5        M)     US-   nSn[        U5       HK  nUUS-  :X  a  US-  S:X  a  S	nU R!                  SUS
9  UUS-  :X  a  US-  S:X  a  S	nU R!                  SUS
9  MM     Xe-
  S-
  n[        U5       H#  nU R#                  5         U R!                  SS	S
9  M%     US-   n[        U5       H  nU R#                  5         M     [(        R*                  R-                  5       (       d   eU R/                  5         U R0                  R3                  5         [4        R6                  R9                  SS9   U R;                  5       nS S S 5        U R                  5         W$ ! , (       d  f       N = f)Nr   r   zself.accumulate_steps=z, num_ranks=Fr
   )r   r   )r   T)r   )r   )r  r  r  )r  r  r  )r  r  )r  r  rg   )r   r-   r*   rm   rX   rS   r6   recv_meta_from_headr+   r4   r  ranger   r   r@   r   r   r   r   r  r   funcs_queueemptyr   r'   !allreduce_shared_weight_gradientsrj   rk   rl   r%  )r7   r  r   rX   compute_lossr,   r*   micro_dataset_phase0micro_dataset_phase1rY   step_1istep_2step_3step_4
have_step5is_last_chunkstep_5step_6r   step_7step_8
train_losss                          r   forward_backward_pipeline+DualPipeVParallel.forward_backward_pipeline  s    NN	!!A%$*?*?9q=*P	5$d##%]	|4	5P(
 	,,T]]D<O<OP##tQ7#tQ7.E "Q&!+vAq.1  1vAq.u5Iq!5577LQ!^  
 q!  !A%vAT2q!q.u=	  &&Q6=A%)A-
 vAEVOMAv..00&&>E '  &&q)'''6&&q)''*//&$(#0#' 0  //&#$(#' 0  //&#0#' 0  ++"#/3/J/J/L ,  ++",# , ] n !A%vA"''1n= 
 	vAFaKD1HM 	Y7FaKD1HM 	Y7  !A%vAT2 
 vA **002222""$668ZZ!!!/335J 0 		 0/s   /R
R(c                    U R                  XU5      nU R                  X5      n[        R                  R	                  SS9   U R                  5         S S S 5        U$ ! , (       d  f       U$ = f)NFrg   )r  r<  rj   rk   rl   _optimizer_step)r7   r  r  r  r   r;  s         r   train_batchDualPipeVParallel.train_batchL  sg     %%d|D33DA
 ZZ!!!/  " 0  0/ s   A
A+)r6   rQ   rP   rK   rJ   rO   rN   rM   rL   rX   r-   rE   rC   rI   rG   r  r3   r4   r1   r*   r  rF   rD   r"   r0   r   rH   rR   r   )rZ   intreturnNone)F)rZ   rB  r   boolrC  rD  )NNF)r   rB  r   rB  rC  rD  )FF)rC  rD  )NTT)rZ   rB  r   rE  r   rE  rC  rD  )FTT)
rZ   rB  r   rE  r   rE  r   rE  rC  rD  )NTFFFF)r   rB  r   rB  r  rE  rC  rD  )FT)NN)$__name__
__module____qualname____firstlineno____doc__r$   r2   r@   rS   r]   rd   rv   r}   r   r   r   rr   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r%  r<  r@  __static_attributes____classcell__)r8   s   @r   r   r   5   s   
>0$5/(1C 7D&	%?"5BMP '+X
X
 X
 
X
v 7<U(	U(n2 %*8$
0 && 	&
 & 
&&  '' ' 	'
 ' 
'( #(4,4, 4,
 4, 
4,l&.( Vx  r   r   )
__future__r   rj   r   2paddle.distributed.communication.batch_isend_irecvr   r    paddle.distributed.communicationr   ImportErrorutils.log_utilr	   pipeline_parallelr   r   r   r   pp_utils.batch_comm_helperr   'pp_utils.forward_backward_overlap_utilsr   zero_bubble_utilsr   r   __all__r   r    r   r   <module>rX     sf   & #  
8 $  8 B :
f( f/  Gs   A A"!A"