
    ϑih                    .   S SK Jr  S SKrS SKJr  S SKJr  S SKJr  S SK	r
S SKrS SKJs  Jr  S SKJr  S SKJr  S SKJr  S S	KJrJr  S S
KJr  S SKJr  S SKJr  S SKJrJrJ r J!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.J/r/  \(       a  SSK0J1r1  / r2\%" 5       r3\ " 5       r4S r5SSS.S jr6 " S S5      r7 " S S5      r8 " S S5      r9 " S S5      r: " S  S!5      r; " S" S#5      r< " S$ S%\<5      r= " S& S'5      r>\S( 5       r? S*     S+S) jjr@g),    )annotationsN)contextmanager)cached_property)TYPE_CHECKING)_C_ops)	ValueDict)grad)core	framework)BuildStrategy)
check_type)switch_to_static_graph)Value
fake_valueget_fake_value_nameis_fake_value   )event_register   )TranslatorLogger)	RETURN_NO_VALUE_MAGIC_NUMBackendCUDAGraphStateTimeCounterauto_layout_is_enabledbackend_guardcse_is_enabledmaybe_dynamic_shape_tensoruse_specialized_device)ConcreteProgramc                F    [        U 5      (       a  [        $ U R                  $ N)r   FAKE_VALUE_NAMEname)values    h/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/jit/dy2static/pir_partial_program.pyget_value_namer'   <   s    U::    T
enable_cseenable_delete_assert_opc                   [         R                  R                  S5      nU(       a  UR                  S0 5        U(       a  UR                  S0 5        UR	                  U 5        g )Nr   %common_subexpression_elimination_passdelete_assert_op_pass)paddlepirPassManageradd_passrun)programr*   r+   pms       r&   apply_general_passesr6   B   sG     
			"B
;R@
+R0FF7Or(   c                  V    \ rS rSrSrS r\S 5       rS rS r	\
S 5       rS rS	 rS
rg)NestSequenceM   z
A wrapper class that easily to flatten and restore the nest structure of
given sequence. It also remove the duplicate variables in the sequence.
For example:
>>> t = [v1, v2, v1]
>>> m = tolist(t)
[v1, v2]
>>> m.restore([t1, t2])
[t1, t2, t1]
c                J    Xl         U R                  5       u  U l        U l        g r"   )
_raw_input_tolist_var_map	_var_list)self	raw_inputs     r&   __init__NestSequence.__init__Y   s    #(,%t~r(   c                    U R                   $ r"   r>   r?   s    r&   var_listNestSequence.var_list]   s    ~~r(   c                    [        5       n/ n[        R                  R                  U R                  5       H?  n[        U[        5      (       d  M  X1;   a  M!  [        U5      X'   UR                  U5        MA     X4$ )zh
Flattens the nested sequences into single list and remove duplicate variables + non-variable elements.
)	r   r/   utilsflattenr;   
isinstancer   lenappend)r?   variable_mapvariable_listr%   s       r&   r<   NestSequence._tolista   sl     !{\\))$//:EeU++$"%m"4L  ' ; **r(   c                $  ^ ^ [        T R                  5      [        T5      :X  d   eU U4S jn[        R                  R	                  T R
                  [        [        U[        R                  R                  T R
                  5      5      5      5      $ )z0
Restores the nested sequence from tensor list.
c                V   > [        U [        5      (       a  TTR                  U       $ U $ r"   )rK   r   r=   )xr?   tensor_result_lists    r&   to_tensor_result.NestSequence.restore.<locals>.to_tensor_resultw   s)    !U##)$--*:;;Hr(   )	rL   r>   r/   rI   pack_sequence_asr;   listmaprJ   )r?   rT   rU   s   `` r&   restoreNestSequence.restoreq   sj     4>>"c*<&====	
 ||,,OO%v||';';DOO'LMN
 	
r(   c                    U R                   n[        U5      S:X  a  US   n[        S U 5       5      (       d   eU Vs/ s H  o R                  U   PM     sn$ s  snf )Nr   r   c              3  B   #    U  H  n[        U[        5      v   M     g 7fr"   )rK   r   ).0vs     r&   	<genexpr>/NestSequence.quick_index_map.<locals>.<genexpr>   s     <A:a''s   )r;   rL   allr=   )r?   
raw_inputsr_   s      r&   quick_index_mapNestSequence.quick_index_map   sV    __
z?a#AJ<<<<<<*45*Qa *555s   Ac                J    U R                    Vs/ s H  o!U   PM	     sn$ s  snf r"   )rd   )r?   tensor_listidxs      r&   quick_restoreNestSequence.quick_restore   s%    ,0,@,@A,@SC ,@AAAs    c                     U R                   U   $ r"   rD   )r?   items     r&   __getitem__NestSequence.__getitem__   s    ~~d##r(   )r;   r>   r=   N)__name__
__module____qualname____firstlineno____doc__rA   propertyrF   r<   rZ   r   rd   ri   rm   __static_attributes__ r(   r&   r8   r8   M   sI    	7  + 
  6 6B$r(   r8   c                  t   \ rS rSrSr\S 5       r\SS j5       r\S 5       r	S r
\S 5       r\S 5       r\S	 5       r\S
 5       r\S 5       r\S 5       r   SS jrS rS rS rS rS rS rS r\S 5       r\S 5       r\    S S j5       r\    S!S j5       r\S 5       r\S 5       r\S 5       rSr g)"RunnableProgram   as  a pir program ready for run_program_op to run. constructed by 3 parts:
- pir program (pir::Program)
- in_out_values
    - input_x values ([string | pir::Value])
    - input_param values ([string | pir::Value])
    - output values ([string | pir::Value])
- forward_backward_ranges
    - forward_range (tuple(Int, Int)) | None
    - backward_range (tuple(Int, Int)) | None
c                H   ^^ / mUU4S jmT" U R                  5       5        T$ )Nc                  > TR                  U R                  5       R                  5       5        U R                   HA  nTR                  UR	                  5       5        UR                  5        H  n T" U 5        M     MC     g r"   )extendkwargsvaluesopsresultsblocks)blockop
all_valuesextend_valuess     r&   r   >RunnableProgram._get_program_all_values.<locals>.extend_values   sZ    elln3356ii!!"**,/YY[E!%( )  r(   )global_block)r4   r   r   s    @@r&   _get_program_all_values'RunnableProgram._get_program_all_values   s&    
	) 	g**,-r(   c                    [         [        5       0n[        R                  U 5       H  nUR                   H  nX!U'   M	     M     U$ r"   )r#   r   rx   r   _names)r4   name_to_value_dictr%   r$   s       r&    _get_name_value_map_from_program0RunnableProgram._get_name_value_map_from_program   sC    0?/N$<<WEE+04( % F "!r(   c                @    [         R                  U R                  5      $ r"   )rx   r   r4   rE   s    r&   name_value_mapRunnableProgram.name_value_map   s    ??MMr(   c                    [        U5      S:X  a  / $ [        US   [        5      (       a  U$ U Vs/ s H  n[        U5      PM     sn$ s  snf Nr   )rL   rK   strr'   )r?   r~   r_   s      r&   convert_nameRunnableProgram.convert_name   sF    v;!IfQi%%M+126aq!6222s   Ac                ^    U R                    Vs/ s H  oR                  U   PM     sn$ s  snf r"   )x_namesr   r?   r_   s     r&   x_valuesRunnableProgram.x_values   s'    04=1##A&===   *c                ^    U R                    Vs/ s H  oR                  U   PM     sn$ s  snf r"   )param_namesr   r   s     r&   param_valuesRunnableProgram.param_values   s+    040@0@A0@1##A&0@AAAr   c                ^    U R                    Vs/ s H  oR                  U   PM     sn$ s  snf r"   )	out_namesr   r   s     r&   
out_valuesRunnableProgram.out_values   s'    04?1##A&???r   c                ^    U R                    Vs/ s H  oR                  U   PM     sn$ s  snf r"   )x_grad_namesr   r   s     r&   x_grad_valuesRunnableProgram.x_grad_values   +    040A0AB0A1##A&0ABBBr   c                ^    U R                    Vs/ s H  oR                  U   PM     sn$ s  snf r"   )p_grad_namesr   r   s     r&   param_grad_values!RunnableProgram.param_grad_values   r   r   c                ^    U R                    Vs/ s H  oR                  U   PM     sn$ s  snf r"   )o_grad_namesr   r   s     r&   out_grad_valuesRunnableProgram.out_grad_values   r   r   Nc                   [        U[        5      (       d   S5       e[        U5      S:X  d   S5       e[        US   [        5      (       d   S5       eXl        U R                  US   5      U l        U R                  US   5      U l        U R                  US   5      U l        X0l	        XPl
        X`l        SU l        SU l        U R                  c4  S[        U R                  R                  5       R                  5      4U l
        U R                  c_  [        U R                  R                  5       R                  5      [        U R                  R                  5       R                  5      4U l        Uc  / / / 4nU R                  US   5      U l        U R                  US   5      U l        U R                  US   5      U l        U R                  S   [        U R                  R                  5       R                  5      :  a4  U R                  R                  5       R                  U R                  S      OS U l        U R                  S   S:  ax  U R                  S   S-
  [        U R                  R                  5       R                  5      :  a7  U R                  R                  5       R                  U R                  S   S-
     OS U l        U R                  S   [        U R                  R                  5       R                  5      :  a:  U R                  R                  5       R                  U R                  S      U l        g S U l        g )Nz)in_out_values must be tuple with len == 3   r   r   r   F)rK   tuplerL   rX   r4   r   r   r   r   out_stop_gradientsforward_rangebackward_rangehas_splitedfinish_passr   r   r   r   r   fwd_end_next_opbwd_start_pre_opbwd_end_nex_op)r?   r4   in_out_valuesr   grad_in_out_valuesr   r   s          r&   rA   RunnableProgram.__init__   s    -// 	
7	
/ =!Q& 	
7	
& -*D11 	
7	
1 ((q)9:,,]1-=>**=+;<"4*,  %"#S)B)B)D)H)H%I!JD&DLL--/334DLL--/334#D %!#R --.@.CD --.@.CD --.@.CD
 !!!$s4<<+D+D+F+J+J'KK LL%%'++D,>,>q,AB 	 ##A&*''*Q.dll//15567 LL%%'++D,?,?,BQ,FG  	 ""1%DLL,E,E,G,K,K(LL LL%%'++D,?,?,BC 	  	r(   c                H   U R                   b  U R                  c  S[        U R                  R	                  5       R
                  5      4U l        [        U R                  R	                  5       R
                  5      [        U R                  R	                  5       R
                  5      4U l        g U R                  S   nU R                  S   nU R                  S   nU R                  S   n[        U R                  R	                  5       R
                  5       H=  u  pVX`R                   :X  a  UnX`R                  :X  a  US-   nX`R                  :X  d  M;  UnM?     U R                  c-  [        U R                  R	                  5       R
                  5      nX4U l        X44U l        g Nr   r   )
r   r   rL   r4   r   r   r   r   	enumerater   )r?   	fwd_startfwd_end	bwd_startbwd_endrh   r   s          r&   update_op_rangeRunnableProgram.update_op_range  s]   '4+@+@+H"#S)B)B)D)H)H%I!JDDLL--/334DLL--/334#D
 **1-I((+G++A.I))!,G$T\\%>%>%@%D%DE---!G... #aI,,,!G F ""*dll779==>"+!5D#,"6Dr(   c                   [         R                  R                  R                  R	                  U R
                  5      u  p[        UU R                  U R                  U R                  4U R                  S U R                  U R                  5      $ r"   )r/   base	libpaddler0   clone_programr4   rx   r   r   r   r   r   r   )r?   cloned_program_s      r&   cloneRunnableProgram.clone,  st    "KK1155CCLL
 \\4++T^^<##
 	
r(   c                   U R                   SL d   S5       eSU l         U R                  5         [        R                  R                  R
                  R                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  5	      u  u  nnnX/U4$ )NFzJPlease ensure only split once! don't call split_forward_backward manually.T)r   r   r/   r   r   r0   split_programr4   r   r   r   r   r   r   r   r   )r?   fwd_progbwd_prog	prog_attrs       r&   split_forward_backward&RunnableProgram.split_forward_backward9  s    5( 	
X	
(   KK!!%%33LLMMOO""  

	
 #Y..r(   c                H   U R                   nU R                  n[        R                  SSU S35        [        R                  SSU S35        U R                  nU" X#U5      u  U l         U l        [        R                  SSU S35        [        R                  SSU S35        g)a|  
Main entries for pass function, without considering any input/output and forward segmentation.
pass_fn' signature is:

1. This function will change forward and backward program.
2. call self.program_attr means start to run.
so we can't call this function after program_attr is called.

def pass_fn(forward_program, backward_program):
    return forward_program, backward_program
r   z<******** [JIT] PIR forward program before PIR PASS ********
 z=******** [JIT] PIR backward program before PIR PASS ********
z;******** [JIT] PIR forward program after PIR PASS ********
z<******** [JIT] PIR backward program after PIR PASS ********
N)forward_programbackward_programprog_loggerlogprogram_name_attr)r?   pass_fn
origin_fwd
origin_bwdr   s        r&   apply_pir_program_pass&RunnableProgram.apply_pir_program_passR  s     ))
**
KJ<WXY	
 	LZLXYZ	

 !226=$57
3d3 	J:,VWX	
 	KJ<WXY	
r(   c                x    U R                   R                  5       R                   H  nUR                  c  M    g   g)NTF)r4   r   r   	dist_attr)r?   r   s     r&   is_distributed_program&RunnableProgram.is_distributed_programw  s1    ,,++-11B||' 2 r(   c                    U R                  5       (       aH  [        R                  R                  R                  R
                  R                  U R                  5        g g r"   )r   r/   distributedauto_parallelstaticmix_to_dist_passapply_mix2dist_passr4   rE   s    r&   "apply_dist_pass_for_origin_program2RunnableProgram.apply_dist_pass_for_origin_program}  sC    &&((,,33DDXX )r(   c                   U R                  5       (       Gad  [        R                  R                  R                  R
                  R                  U R                  5        [        R                  R                  R                  R                  R                  U R                  5        [        R                  R                  R                  R                  R                  R                  U R                  5        [        R                  R                  R                  R                  U R                  5        [        R                  R                  R                  R                  R!                  U R                  5        g g r"   )r   r/   r   r   r   r   r   r4   pir_passapply_partition_passReshardPassesapply_reshard_passr   r   r0   apply_dist2dense_passremove_unuseful_comm_op_passrE   s    r&   !apply_dist_pass_for_whole_program1RunnableProgram.apply_dist_pass_for_whole_program  s    &&((,,33DDXX ,,33<<QQ ,,33<<JJ]] KK!!%%;;DLLI,,33<<YY )r(   c                "    U R                  5       $ r"   )r   rE   s    r&   _forward_backward_program)RunnableProgram._forward_backward_program  s    **,,r(   c           	        U R                   SL d   S5       eSU l         [        R                  U R                  5      nU R                  nUS   n0 n[        R                  U R                  U5      n[        R                  U R                  U5      nUR                  5        HI  u  pVXV1[        U5      -  (       d  M  XS;   a  UR                  U5        Xc;   d  M8  UR                  U5        MK     [        R                  U R                  U5        0 nU R                  R                  5        HA  u  pUS:X  a/  U	 V
s/ s H  oR                  U
[        5       5      PM     sn
Xx S3'   XU S3'   MC     [        US   5      [        U R                  5      :X  d   S	5       e[        US   U R                  5       H  u  p[!        U5      (       a  M  Xl        M     U$ s  sn
f )
NFzgprogram_attr() is called by PartialProgramLayer, don't call it manually, use program_name_attr instead.Tno_need_buffersfo_valuesr   	fo_valuesz0Output values and stop gradients length mismatch)r   rx   r   r   r   unify_value_namesr   itemssetremoveupdate_program_name_attrgetr   rL   r   zipr   stop_gradient)r?   fwd_mapr   no_need_buffer_namesrename_mappingoriginal_namenew_nameprogram_attrknsnr_   r  s                r&   r  RunnableProgram.program_attr  s   5( 	
u	
(  !BB  
 !2201BC(::  .
 )::!!>
 (6';';'=#M(3/C+DDD 8(//>3(//9 (> 	00""N	
 ++113EADy:</:<QKK:<0"/s']+ *,A3f& 4 <,-T5L5L1MM 	
>	
M !$%t'>'>!
A Q+O!
 !/s   8#Gc                   [        U5      n[        R                  U 5       Ha  nUR                  (       d  M  UR                  nUR                  X35      nUR                  UR                  X0R                  5       5      5        Mc     [        R                  U 5       HB  nUR                  (       d  M  UR                  5       (       a  M-   SU SUR                   35       e   U$ )z7Ensure every value at most has one name in the program.z7Expected all values in Program have only one name, but z has multiple names: )dictrx   r   has_namer$   r  update_renamer   _has_only_one_namer   )r4   r  r%   r  s       r&   r  !RunnableProgram.unify_value_names  s    
 n-$<<WEE>>zzH%))(=H!!h(<(<(>? F %<<WEE~~~//11 MeWTijojvjviwx1 F
 r(   c                |    U R                  5        H#  u  p#U Vs/ s H  oDU;   a  X   OUPM     snX'   M%     g s  snf r"   )r  )	name_attrr  r  vsr_   s        r&   r  (RunnableProgram.update_program_name_attr  sD     __&EAIKIKA.%8!a?IL 's   9c                    U R                   S   n0 nUR                  5        H#  u  p4U Vs/ s H  n[        U5      PM     snX#'   M%     U$ s  snf Nr   )r   r  r'   )r?   origin_attr_program_attrr  r  r_   s         r&   r   !RunnableProgram.program_name_attr  sS    44Q7 &&(EA;=>2aq 12>M )  ?s   Ac                &    U R                   S   S   $ r   r   rE   s    r&   r   RunnableProgram.forward_program      --a033r(   c                &    U R                   S   S   $ r   r$  rE   s    r&   r    RunnableProgram.backward_program  r&  r(   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r4   r   r   )returnzdict[str, Value]NNN)r  dict[str, str]r)  r+  )r  zdict[str, list[str]]r  r+  )!ro   rp   rq   rr   rs   staticmethodr   r   r   r   r   r   r   r   r   r   r   rA   r   r   r   r   r   r   r   r   r  r  r  r   r   r   ru   rv   r(   r&   rx   rx      s   	   " " N N3 > > B B @ @ C C C C C C  ;
z74
/2#
J" - - 2 2h !/	 * '9G    4 4 4 4r(   rx   c                  &    \ rS rSrS rS rS rSrg)PartialProgramLayerHooki  c                    X4$ r"   rv   )r?   r   src_varss      r&   before_append_backward.PartialProgramLayerHook.before_append_backward  s    ((r(   c                
    XU4$ r"   rv   )r?   whole_programinputsr0  grad_outputsforward_end_idxbackward_start_idxs          r&   after_append_backward-PartialProgramLayerHook.after_append_backward  s     x77r(   c                    U$ r"   rv   )r?   infer_programs     r&   after_infer#PartialProgramLayerHook.after_infer  s    r(   rv   N)ro   rp   rq   rr   r1  r9  r=  ru   rv   r(   r&   r.  r.    s    )	8r(   r.  c                  (    \ rS rSrSrSrS rS rSrg)OperatorIndexPreservePassi  preserved_index_r   c                    [         R                   [         R                   3U l        [         =R                  S-  sl        X l        Xl        g r  )r@  OP_NAME_PREFIXcounterr$   r   index)r?   rE  r   s      r&   rA   "OperatorIndexPreservePass.__init__  s<    0??@AZAbAb@cd	!))Q.)
r(   c                b   [        UR                  5       R                  5      S:X  a#  U R                  S:X  d   eU R	                  U5      $ [
        R                  R                  R                  R                  UUR                  5       R                  S   R                  S5      U R                  U R                  5        U R	                  U5      nSnUR                  5       R                   H?  nUR                  5       S:X  a#  U R                  UR                  5       S   ;   a    O	US-  nMA     U[        UR                  5       R                  5      :  a  [        SU R                   S35      eUR                  5       R                  UR                  5       R                  U   5        X l        U$ )Nr   builtin.shadow_outputoutput_namer   z Can't find index preserve label z, don't remove it in pass.)rL   r   r   rE  r   r/   r   r   r0   append_shadow_outputresultr$   attrsRuntimeError	remove_op)r?   r4   	new_indexr   s       r&   __call__"OperatorIndexPreservePass.__call__  sa   w##%))*a/::?"?<<((!!66  "&&q)003IIJJ		
 ,,w'	&&(,,B	44IIM!::NI - G00266772499+=WX  	(()=)=)?)C)CI)NO
r(   )rE  r$   r   N)	ro   rp   rq   rr   rC  rD  rA   rP  ru   rv   r(   r&   r@  r@    s    'NGr(   r@  c                       \ rS rSrS rS rSrg)IndicesPreservePassi7  c                *    X l         Xl        S U l        g r"   )r   indicesnew_indices)r?   rU  r   s      r&   rA   IndicesPreservePass.__init__8  s    r(   c                    U R                   /n[        U R                  5       H"  u  p4UR                  [	        XBU   5      5        M$     US   " U5      nUSS   Vs/ s H  ofR
                  PM     snU l        U$ s  snf )Nr   )r   r   rU  rM   r@  rE  rV  )r?   r4   passesrh   rE  new_programps          r&   rP  IndicesPreservePass.__call__=  sp    ,,#DLL1JCMM3E#;GH 2Rj)-3ABZ8ZGGZ8 9s   A:)rU  rV  r   N)ro   rp   rq   rr   rA   rP  ru   rv   r(   r&   rS  rS  7  s     
r(   rS  c                  J    \ rS rSrSrS rS r\S 5       r\S 5       r	S r
Srg	)
ValuePreservePassiG  preserved_value_c                    Xl         X l        g r"   )r~   use_cinn_pass)r?   r~   rb  s      r&   rA   ValuePreservePass.__init__J  s    *r(   c                    [        S5      e)NzNot implemented.)rM  )r?   r4   s     r&   applyValuePreservePass.applyN  s    -..r(   c                   ^ ^ SmUU 4S jnU$ )Nr   c                    > T T 3n TS-  mU $ r  rv   )r$   countprefixs    r&   name_gen9ValuePreservePass.create_name_generator.<locals>.name_genU  s    XeW%DQJEKr(   rv   )rj  rk  ri  s   ` @r&   create_name_generator'ValuePreservePass.create_name_generatorQ  s    	 r(   c           	         [        U 5      (       a  g X;   a  X    $ U" 5       nXBU '   [        R                  R                  R                  R                  UU U[        UR                  5       R                  5      5        U$ r"   )	r   r/   r   r   r0   rJ  rL   r   r   )r%   r4   
value2namename_generatorr$   s        r&   attach_preserved_name'ValuePreservePass.attach_preserved_name]  st    $$ 5!!66$$&**+		
 r(   c                  ^^^^ [        5       m[        R                  [        R                  5      m[        R
                  R                  UUU4S jU R                  5      n@U R                  T5      m0 m/ nTR                  5       R                   H  nUR                  5       S:X  d  M  UR                  5       S   R                  [        R                  5      (       d  MP  UR                  S5      R                  5       TUR                  5       S   '   UR!                  U5        M     U H"  nTR                  5       R#                  U5        M$     [        R
                  R                  U4S jU5      U l        T$ )Nc                4   > [         R                  U TTT5      $ r"   )r_  rr  )r%   rq  r4   rp  s    r&   <lambda>,ValuePreservePass.__call__.<locals>.<lambda>t  s    +AA	r(   rH  rI  r   c                8   > TR                  U [        5       5      $ r"   )r  r   )r$   name2new_values    r&   rv  rw    s    ++D*,?r(   )r   r_  rm  rC  r/   rI   map_structurer~   re  r   r   r$   rL  
startswithoperandsourcerM   rN  )r?   r4   namesto_remove_opr   ry  rq  rp  s    `   @@@r&   rP  ValuePreservePass.__call__m  s>   [
*@@,,
 ** KK
  **W% &&(,,Bwwy3388:m,77%44  AC

Afh #288:m#<= !''+ - B  ",,R0  ll00?
 r(   )rb  r~   N)ro   rp   rq   rr   rC  rA   re  r,  rm  rr  rP  ru   rv   r(   r&   r_  r_  G  s>    'N+/ 	 	  )r(   r_  c                      \ rS rSrS rSrg)FullGraphPreProcessPassi  c                   [         R                  R                  R                  R	                  U5      nU R
                  (       a  [         R                  R                  R                  R                  5       nUR                  S0 5        [         R                  R                  R                  R                  X!5        [         R                  R                  R                  R                  X!5        UR                  U5        U$ )Nr.   )r/   r   r   r0   apply_bn_add_act_passrb  r1   r2   infer_symbolic_shape_passreduce_as_sum_passr3   )r?   r4   r5   s      r&   re  FullGraphPreProcessPass.apply  s    ++''++AA'J&&**668BKK/4KK!!%%??LKK!!%%88EFF7Or(   rv   N)ro   rp   rq   rr   re  ru   rv   r(   r&   r  r    s    	r(   r  c                    ^  \ rS rSrSrSr S"SS.U 4S jjjr\S 5       rS r	\
" S5      S	 5       rS
 r\S#S j5       rS rS$S jr\S%S#S jj5       r\S 5       r\S 5       r\S#S j5       r\S 5       r\S#S j5       r\S#S j5       rS r\    S&S j5       rS%S jrS rS$S jrS r\S 5       r S r!S r"S'S jr#S r$\S  5       r%S!r&U =r'$ )(PartialProgramLayeri  a  
PartialProgramLayer wraps all the ops from layers decorated by `@to_static`
and execute them as a static subgraph.

.. note::
    **1. This is a very low level API. Users should not use this API
         directly. Please use `partial_program_from(concrete_program)`
         to create it.
    **2. TensorArray is not currently supported in the output.

Args:
    main_program(Program): The main program that contains ops need to be executed.
    inputs(list[Variable]): The input list of the decorated function by `@to_static`.
    outputs(list[Variable]): The output list of the decorated function by `@to_static`.
    parameters(list[Tensor]|None): All trainable parameters included in the program. Default None.
    constraints(list[tuple[str, int|None, int|None]]): A list to specify the constraints of the program. Default None.

Returns:
    Layer: A Layer object that run all ops internally in static graph mode.
N)constraintsc               p  > [         TU ]  5         [        U5      U l        [        U5      U l        Ub  UO/ U l        Ub  UO/ / 4u  U l        U l        UR                  S[        5       5      U l
        [        U R                  [        5      (       d   eU R                  XR                  5      U l        Ub$  U R                  US   S S & U R                  US   S S & SU l        0 U l        Su  pxn	[         R"                  " 5       n
U
(       a  U
R%                  5       u  pU
R&                  nUb>  US;   a8  [(        R*                  R,                  R.                  R1                  UU	US9U l        0 U l        / U l        UR                  S[8        R:                  5      U l        0 U l        [A        5       U l!        0 U l"        g )	Nbuild_strategyr   r   Tr*  )float16bfloat16)custom_white_listcustom_black_listdtypebackend)#superrA   r8   _inputs_outputs_constraints_params_param_valuesr  r   _build_strategyrK   _verify_program_origin_main_programtraining_program_extra_infor   _dygraph_tracer_get_amp_op_list
_amp_dtyper/   r   amp
fp16_listsAutoMixedPrecisionLists	_amp_list_scope_cache_hookersr   PHI_backend_grad_var_namesr   _compile_time_counter_prog_attrs_map_cache)r?   main_programr5  outputs
parametersr  r}   	amp_dtyper  r  tracer	__class__s              r&   rA   PartialProgramLayer.__init__  s    	#F+$W-+6+BK$0Jr2h 	)d(  &zz*:MOL$..>>>>$($8$8--%
! !#||JqM!#11JqM!#% :J7	&7**,393J3J3L0))I Y2I%I !!,,DD&7&7# E  N 

9gkk:!%0]"%'"r(   c           	     z   Uu  pE[         R                  R                  R                  [         R                  R                  R	                  U5      UUS   [
        R                  :g  US   5      n[        R                  " [        R                  U5      [        R                  U5      U R                  USS9UU5      $ )Ncuda_graph_statecuda_graph_dispatch_keyT	cache_keyuse_scope_cache)r/   r   r
   calc_scope_cache_keyget_program_id_from_attrsr   DISABLEr   run_programr  _valid_vars_create_scope_vec)partial_program_layerr5  r  rL  
prog_attrscuda_graph_attrsscope_cache_keys          r&   run_implPartialProgramLayer.run_impl  s    ',$
 ++**??KK66zB/0N4J4JJ67	
 !!++F3++J7!33) $ 4  	
 		
r(   c                    U R                  SS9nU R                  U5      nU R                  UU R                  U5      nU R	                  U5      nU R                  U5      $ )zA
Execute static graph by Interpreter and Return dynamic Tensors.
Fin_sot_mode)_prepare_attributes_prepare_inputscall_run_impl_with_hookr  _restore_out_remove_no_value)r?   r5  rL  outrestored_nest_outs        r&   rP  PartialProgramLayer.__call__	  si     ((U(;%%f-**LL
 !--c2$$%677r(   zsot call partial_programc                    U R                  SS9nU R                  UU R                  U5      nU R                  R	                  U5      $ )zj
In sot, inputs and outputs of partial program only contain tensors, so we can skip some step to speed up
Tr  )r  r  r  r  ri   )r?   r5  rL  r  s       r&   sot_callPartialProgramLayer.sot_call  sK    
 ((T(:**LL

 }}**3//r(   c                    [         R                  c'  [         R                  R                  U 5      " UUU5      $ [         R                  [         R                  R                  U 5      UUU5      $ r"   )r  HOOKED_RUN_IMPLr  __get__)r?   r5  r  rL  s       r&   r  +PartialProgramLayer.call_run_impl_with_hook'  si     ..6&//77=  '66#,,44T:	 r(   c                   [        U R                  R                  5      n[        U R                  R                  5      nU Vs/ s H  o3R                  PM     nnU R
                  n[        R                  R                  R                  R                  U R                  U[        U R                  R                  5       R                  5      S5        [        U R                  XU4U5      $ s  snf )Noutput_)rX   r  rF   r  r  r  r/   r   r   r0   append_shadow_outputsr  rL   r   r   rx   )r?   r5  r  r_   r   paramss         r&   origin_runnable_program+PartialProgramLayer.origin_runnable_program;  s    dll++,t}}--. 8??w!oow?##!!77%%))668<<=		
 %%W%
 	
 @s   C*c                :    U R                   R                  U5        g r"   )r  rM   )r?   hookers     r&   
add_hookerPartialProgramLayer.add_hookerO  s    V$r(   c                    U(       d  [         R                  " 5       $ XR                  ;  a  / U R                  U'   U R                  U   nU H  nUR                  (       d  M  Us  $    [         R                  " 5       nUR	                  U5        U$ r"   )r
   Scoper  _can_reusedrM   )r?   r  r  cached_scopesscopes        r&   
_get_scopePartialProgramLayer._get_scopeR  sx    ::<---+-Di()))4"E    # 

U#r(   c                  ^  U(       a  U 4S jnT R                   R                  5       n[        5       (       ak  T R                  R	                  5       (       aL  [
        R                  R                  S5      nUR                  S0 5        UR                  UR                  5        T R                   H  nUR                  U5        M     UR                  U5        U$ T R                   R                  5       nUR                  5         [        5       (       ak  T R                  R	                  5       (       aL  [
        R                  R                  S5      nUR                  S0 5        UR                  UR                  5        T R                  U5      nT R!                  T R"                  U5        U 4S jnUR                  U5        U$ )Nc                h  > TR                   R                  5       (       a  [        U [        5       TR                   R                  5       S9  [        R
                  R                  R                  R                  U TR                  5        [        R
                  R                  R                  R                  U 5        X4$ TR                   R                  5       (       at  [        R
                  R                  R                  R                  U TR                  5        [        R
                  R                  R                  R                  U 5        X4$ [        U [        5       TR                   R                  5       S9  [        R
                  R                  R                  R                  U 5        X4$ )Nr)   )r  is_cinnr6   r   r/   r   r   r0   bind_symbolic_constraintsr  apply_cinn_passis_pccapply_pcc_passcheck_infer_symbolic_if_need)r   r   r   r?   s      r&   r   4PartialProgramLayer._create_program.<locals>.pass_fnd  sG   ==((**('#1#3040E0E0G
 KK))--GG'):): KK))--==oN  '88 ]]))++KK))--GG'):): KK))--<<_M '88 )'#1#3040E0E0G
 KK))--JJ' '88r(   r   auto_layout_passc                  >^ U4S jn[        U [        5       TR                  R                  5       S9  [        U[        5       TR                  R                  5       S9  TR                  R                  5       (       a  [        R
                  R                  R                  R                  U TR                  5        [        R
                  R                  R                  R                  U 5        U" X5        [        R
                  R                  R                  R                  U5        X4$ TR                  R                  5       (       at  [        R
                  R                  R                  R                  U TR                  5        [        R
                  R                  R                  R                  U 5        X4$ [        R
                  R                  R                  R                  U 5        X4$ )Nc                  >^	^
^ [         R                  R                  R                  R	                  U 5      m[         R                  R                  R                  R	                  U5      m	T	R                  T5        U R                  5        VVs0 s H  nUR                    H  nX2_M     M     snnm
U	U4S jnU
U4S jnUR                  5       R                  5       R                  5        H  u  pgU" Xg5      nU" X5        M     g s  snnf )Nc                H   > TR                  UTR                  U 5      5        g r"   )set_shape_or_data_for_varget_shape_or_data_for_var)forward_valuebackward_valuebackward_shape_analysisforward_shape_analysiss     r&   +share_symbol_shape_from_forward_to_backwardڗPartialProgramLayer._create_program.<locals>.pass_fn.<locals>.init_backward_program_shape_analysis.<locals>.share_symbol_shape_from_forward_to_backward  s'     0II*2LL -r(   c                   > U TS   ;   a  TS   R                  U 5      nTTS   U      $ U T;   a  TU    $ [        SU  S35      e)Nbo_gr   z	kw_args: z
 not found)rE  	Exception)kw_namekw_valuerh   forward_name_value_mapr   s      r&    get_kwargs_forward_matched_valueڌPartialProgramLayer._create_program.<locals>.pass_fn.<locals>.init_backward_program_shape_analysis.<locals>.get_kwargs_forward_matched_value  sj    "&7&??"3F";"A"A'"JC#9 1$ 7 <$  %(>>#9'#BB"+iy
,K"LLr(   )r/   r   r   r0    get_shape_constraint_ir_analysis(register_symbol_cstr_from_shape_analysis	list_varsr   r   r}   r  )r   r   rl   r$   r  r  r   r  forward_matched_valuer  r  r  r   s            @@@r&   $init_backward_program_shape_analysisbPartialProgramLayer._create_program.<locals>.pass_fn.<locals>.init_backward_program_shape_analysis  s     .4[[-B-B-F-F-g-g'.* /5kk.C.C.G.G.h.h(/+ ,TT.
 %4$=$=$?.$?D$(KKD 
$/ $?.*	M )557>>@FFH , =WO . D1 I9.s    Dr)   )r6   r   r  r  r/   r   r   r0   r  r  r  r  r  r  )r   r   r   r	  r?   s     ` r&   r   r    s{   /b %#-/,0MM,A,A,C
 %$-/,0MM,A,A,C
 ==((**KK))--GG'):): KK))--==oN8' KK))--==>NO '88 ]]))++KK))--GG'):): KK))--<<_M
 '88 KK))--JJ' '88r(   )r  r   r   r  r  r/   r0   r1   r2   r3   r4   r  r=  r   r   _append_backward_set_grad_typer  )r?   is_infer_moder   r<  r5   r  train_programs   `      r&   _create_program#PartialProgramLayer._create_program`  s[   9< !88>>@M%''DMM,A,A,C,CZZ++A..3},,---""=1 (009   ,,224  <<> &''DMM,A,A,C,CZZ++A..3},,- 11-@Mm<O9b 009  r(   c                Z    [         R                  R                  U R                  U 5      nU$ r"   )r/   rI   _hash_with_idr  )r?   
program_ids     r&   _train_program_id%PartialProgramLayer._train_program_id  s$    \\//0B0BDI
r(   c                V    [         R                  R                  U R                  U 5      $ r"   )r/   rI   r  r<  rE   s    r&   _infer_program_id%PartialProgramLayer._infer_program_id  s    ||))$*<*<dCCr(   c                T    U R                   (       a  U R                  $ U R                  $ )z'
Return current train or eval program.
)r  r  r<  rE   s    r&   r4   PartialProgramLayer.program  s#    
 ==%%%%%%r(   c                T    U R                   (       a  U R                  $ U R                  $ )z/
Return current train or eval program hash id.
)r  r  r  rE   s    r&   r  PartialProgramLayer.program_id  s#    
 ==))))))r(   c                    [        U R                  5         U R                  R                  5          U R	                  5       sS S S 5        sS S S 5        $ ! , (       d  f       O= fS S S 5        g ! , (       d  f       g = fr"   r   r  r  recordr  rE   s    r&   r  !PartialProgramLayer.train_program  sK    4==)4+E+E+L+L+N'') ,O+N))+N+N)))s"   A.A	A.
A"	A..
A<c                    [        U R                  5         U R                  R                  5          U R	                  SS9sS S S 5        sS S S 5        $ ! , (       d  f       O= fS S S 5        g ! , (       d  f       g = f)NT)r  r  rE   s    r&   r<  !PartialProgramLayer.infer_program  sP    4==)4+E+E+L+L+N''d'; ,O+N))+N+N)))s"   A-A 	A-
A!	A--
A;c                (    U R                  U5        U$ )zp
Verify that the program parameter is initialized, prune some unused params,
and remove redundant op callstack.
)_check_params_all_inited)r?   r  r  s      r&   r  #PartialProgramLayer._verify_program  s     	%%l3r(   c                   UR                   nUR                  nU R                   H  nUR                  X#5      u  p#M     UR                  nUR
                  n[        [        R                  " XV5      5      n[        UR                  5       R                  5      nS n	US:  a  UR                  5       R                  S   n	S /[        U5      -  n
[        U R                  5         [        US[        [        [         4S5        ["        R$                  " US 5         / nU H^  nUR&                  SL a  UR)                  [+        5       5        M-  [,        R.                  " USUR0                  S9nUR)                  U5        M`     [,        R2                  R4                  R6                  R9                  UU[        UR                  5       R                  5      S5        [        UR                  5       R                  5      U-
  n[        [        [;        S	 U5      5      5      S:  ap  [=        U[        [;        S
 U5      5      [        [;        S U5      5      S9n
U	b:  [?        UR                  5       R                  5       H  u  nnUU	:X  d  M  US-   n  O   S S S 5        U R                   H   nURA                  UUUWUUW-   5      u  nnnM"     S S S 5        S n[        U5      n[        [C        UU
SU 5      5      n[        [C        UU
US  5      5      n[        [C        UW5      5      n[        [;        S U5      5      n[        [;        S UU-   5      5      n[        UR                  5       R                  5      n[,        R2                  R4                  R6                  R9                  UUUS5        UW-   n[E        XVUUUU/U R                  RG                  5       5      n[I        UUU/U5      nU" U5      nURJ                  u  nnnnnnURL                  u  nnn[O        UXVU4URP                  UUU4SU4UU45      nURS                  5         U$ ! , (       d  f       GN= f! , (       d  f       GN= f)Nr   rY  targetszpaddle.static.gradientsTg      ?)
fill_valuer  grad_input_c                    U R                   SL $ NFr  rS   s    r&   rv  6PartialProgramLayer._append_backward.<locals>.<lambda>R      Q__-Er(   c                    U R                   SL $ r+  r,  r-  s    r&   rv  r.  Z  r/  r(   c                "    [        U 5      (       + $ r"   r   r-  s    r&   rv  r.  ^  s    mA.>*>r(   )r5  r  r6  r   c                D    [        U [        5      (       a  U $ [        5       $ r"   )rK   r   r   r-  s    r&   rv  r.  w  s    z!U';';!"M"Mr(   c                "    [        U 5      (       + $ r"   r2  r-  s    r&   rv  r.        q!11r(   c                "    [        U 5      (       + $ r"   r2  r-  s    r&   rv  r.    r5  r(   grad_output_)*r4   r   r  r1  r   r   rX   	itertoolschainrL   r   r   r   r  r   r   r   	ir_staticprogram_guardr  rM   r   r/   	full_liker  r   r   r0   r  filterr	   r   r9  rY   r  r  rS  r~   rV  rx   r   r   )r?   train_runnable_programr4   r'  r  r5  r  combined_inputsr7  forward_end_opgrad_info_mapforward_outputs_grads	out_valuer%   op_between_forward_and_backwardrh   r   mapping_valueinputs_sizex_grad_valuep_grad_valueo_grad_valueinput_grads_to_appendoutput_grads_to_appendbackward_end_op_indexbackward_start_op_indexfull_graph_pre_process_passforward_index_passr4  s                                r&   r  $PartialProgramLayer._append_backward  so    )00(33mmF%<<WNGW $'00'44yv>?g224889Q$11377;N_!554==)e$)	 (($7(*%!(I ..$6-44Z\B & 0 0%'*"+//!
 .44U; ") %%))??),,.223!	 ,,.223oE 0 "#EwO
  %). $"#EwO! &*" > 5&%M &1'01E1E1G1K1K'LGC!^325' % (M[ 8d --
 00#)##&EE	#	 (s *N N&kC}Q{/KLMC}[\/JKLC/DEF !%1<@!
 "&1<,3NO"
 !$G$8$8$:$>$> ?!!77"!		
 == 	 
 '>WlL,OMM!!#'
# 157LM'

 %W- (..	
 **		
#! (W%"55<6 $&;<
 	779w 87 *)s+   "5QE*Q	Q8Q
Q	Q
Q'c                   U R                   U R                  U4nX R                  ;  a  U R                  R                  U R                  R
                  U R                  (       + U R                   US.U R                  R                  -  n[        R                  R                  R                  U5      U R                  U'   [        R                  SS.nU R                  U   U4$ )N)r   r   is_testr  r  r   )r  r  )r  r  r  r4   r   r   r  r/   r   r
   construct_program_attribute_mapr   r  )r?   r  prog_attr_keyr  r  s        r&   r  'PartialProgramLayer._prepare_attributes  s    $--E : ::#'<<#?#?$(LL$A$A#}},"oo* ))*J   @@L &&}5
 !/ 6 6'(
 ))-8:JJJr(   c                   [        U[        [        45      (       d   e[        R                  R                  U5      n/ n[        R                  " 5       n[        U5       H  u  pV[        U[        R                  5      (       a#  Sn[        R                  R                  USUSS9nO[        U[        R                  R                  5      (       am  UR                  (       aY  UR                  R!                  U5      (       d9  [#        5       (       d*  [%        U5      (       d  UR'                  US5      nSUl        OUnOM  UR)                  U5        M     U$ )z!
Prepare inputs, outputs, attrs.
NFT)r%   persistableplace	zero_copy)rK   r   rX   r/   rI   rJ   r   _current_expected_placer   npndarrayr
   eagerTensorr  rX  _equalsr   r   _copy_torM   )r?   r5  flatten_inputs
input_varsexpected_placeir%   vars           r&   r  #PartialProgramLayer._prepare_inputs  s    &5$-0000--f5
"::<!.1HA%,,jj'' %("	 (  E4::#4#455 ''!KK//??2446u==..?C(,C%Cc"1 22 r(   c                &    U R                  XS9nU/$ )Nr  )r  )r?   r  r  inner_scopes       r&   r  %PartialProgramLayer._create_scope_vec  s"    oo & 
 }r(   c                j    U R                   R                  U5      nUb  [        U5      S:X  a  US   nU$ )zJ
Restores same nested outputs by only replacing the Variable with Tensor.
r   r   )r  rZ   rL   )r?   out_varsoutss      r&   r   PartialProgramLayer._restore_out  s6     }}$$X.D	Q7Dr(   c                     UR                  SS9$ )NT)for_test)r   )r?   r  s     r&   _clone_for_test#PartialProgramLayer._clone_for_test  s    !!4!00r(   c                    [        U[        R                  R                  5      (       a-  UR                  S/:X  a  UR                  5       S   [        :X  a  gg)Nr   r   TF)rK   r
   r]  r^  shapenumpyr   )r?   re  s     r&   _is_no_value PartialProgramLayer._is_no_value   s?    c4::,,--#))s2Byy{1~!::r(   c                  ^  [        U[        R                  R                  5      (       a  T R	                  U5      (       a  gU$ [        U[
        [        45      (       a  [        U[
        5      (       a  [        U 4S jU 5       5      nO*U Vs/ s H  nT R	                  U5      (       a  M  UPM     nn[        U5      [        U5      :  n[        U5      S:X  a  U(       a  g[        U5      S:X  a  U(       a  US   $ U$ U$ s  snf )z;
Removes invalid value for various-length return statement
Nc              3  X   >#    U  H  nTR                  U5      (       a  M  Uv   M!     g 7fr"   )ru  )r^   re  r?   s     r&   r`   7PartialProgramLayer._remove_no_value.<locals>.<genexpr>  s#      #+C43D3DS3ICC8s   *	*r   r   )rK   r
   r]  r^  ru  r   rX   rL   )r?   rk  resre  has_removeds   `    r&   r  $PartialProgramLayer._remove_no_value  s     h

 1 122  **O5$-00(E** #+  '/Mhsd6G6G6LshMh-#c(2K 3x1}SQ;1vJ Ns   D,Dc                
   UR                   nUR                  n[        X5       H  u  pE[        U5      (       a  M  UR	                  5       (       aI  UR                  [        R                  R                  R                  R                  R                  5        Mu  UR                  5       (       aI  UR                  [        R                  R                  R                  R                  R                  5        M  [        S5      e   g )Nz5only support selected_row and dense_tensor grad type.)r   r4   r  r   is_selected_row_typer  r/   r   r
   VarDescVarTypeSELECTED_ROWSis_dense_tensor_typeDENSE_TENSORNotImplementedError)r?   r  r  forward_params_gradsparamr%   s         r&   r  "PartialProgramLayer._set_grad_type"  s      ->>%--=LEU##))++$$KK$$,,44BB ++--$$KK$$,,44AA *K  >r(   c           	        [        U R                  [        [        45      (       d"  [	        S[        U R                  5       S35      e[        5       n[        U R                  5       Hd  u  p4[        U[        R                  R                  5      (       d  [	        SU S[        U5       S35      eUR                  UR                  5        Mf     g)a]  
Check all params from main program are already initialized, see details as follows:
    1. all parameters in self._params should be type `framework.EagerParamBase` which are created in dygraph.
    2. all parameters from transformed program can be found in self._params.
       Because they share same data with EagerParamBase of original dygraph.
zRType of self._params in PartialProgramLayer should be list or tuple, but received .zType of self._params[zG] in PartialProgramLayer should be Parameter or Variable, but received N)rK   r  rX   r   	TypeErrortyper  r   r
   r]  r^  addr$   )r?   r  param_and_buffer_names_setrd  re  s        r&   r$  ,PartialProgramLayer._check_params_all_inited;  s     $,,u66deijnjvjvewdxxyz  &)U"-FAc4::#4#455+A3.uvz{~v  vA  AB  C  '**3884 .r(   c                    U (       a  U $ S $ r"   rv   )varss    r&   r  PartialProgramLayer._valid_varsP  s    t%%r(   )r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"   )r)  rx   r+  F)r>  rx   r)  rx   )r  rx   )(ro   rp   rq   rr   rs   r  rA   r,  r  rP  r   r  r  r   r  r  r  r   r  r  r  rt   r4   r  r  r<  r  r  r  r  r  r  rp  ru  r  r  r$  r  ru   __classcell__)r  s   @r&   r  r    s   * O 5( 5( 5(n 
 
&8  ./0 00( 
 
&% K! K!Z   D D & & * * * * < < R&5R	R RhK(#J 1 1625* & &r(   r  c              #     #    [         R                  nU [         l         Sv   U[         l        g! U[         l        f = f7f)zr
A context manager to temporarily replace the run_impl of PartialProgramLayer.
This is used for testing purposes.
N)r  r  )new_run_implold_run_impls     r&   replace_run_impl_guardr  U  s5      '66L*6';.:+l+s   ?/ ?<?c                    U R                   nU(       a  U(       a  USS  n[        U R                  UU R                  U R                  4SU R
                  0U R                  D6$ )Nr   r  )r5  r  r  r  r  r  r}   )concrete_programfrom_methodr5  s      r&   partial_program_fromr  c  sl     $$F +%%  ##	
 %00 
!
! r(   r  )r  r    r  boolr)  r  )A
__future__r   r8  
contextlibr   	functoolsr   typingr   rt  r[  r/   paddle.pir.corer0   r
   r:  r   paddle.autograd.backward_utilsr   paddle.autograd.ir_backwardr	   paddle.baser   paddle.base.compilerr   paddle.base.data_feederr   paddle.base.dygraph.baser   
paddle.pirr   r   r   r   profilerr   logging_utilsr   rI   r   r   r   r   r   r   r   r   r   program_translatorr    __all__r   r#   r'   r6   r8   rx   r.  r@  rS  r_  r  r  r  r  rv   r(   r&   <module>r     s   #  % %     # #  4 , ' . . ; L L % +
 
 
 3
  &'  @$ @$Fj4 j4Z &$ $N  O Od
/ 
l
& l
&^ 
; 
; <A%48r(   