
    x-j                        d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ dZdZdZd	Zej                                        Zej                                        Zej        j        j        Zej        j        Zej                                        Zej        j        j        Zej        j        j        Zej        j        j        Zej                                         Z!g d
Z"g dZ#ddgZ$dddZ%dddZ&dZ'g dZ(d e(D             Z)d Z*dZ+ e*dd          Z, G d d          Z- G d d          Z.d Z/g fdZ0d Z1d  Z2d! Z3d" Z4d# Z5d$ Z6d% Z7d& Z8d' Z9d( Z:d) Z;d* Z<d+ Z=d, Z>	 d`d.Z?d/ Z@d0 ZAdad1ZBdbd2ZCd3 ZDd4 ZEd5 ZFd6 ZGd7 ZHd8 ZI	 dcd:ZJd; ZKd< ZLd= ZMd> ZNd? ZOd@ ZPdA ZQ	 dddCZRdedDZSdE ZTejU        jV        jW        dFejU        jV        jX        dGejU        jV        jY        dHejU        jV        jZ        dFejU        jV        j[        dGejU        jV        j\        dHejU        jV        j]        dIejU        jV        j^        dIiZ_dJ Z` G dK dL          ZadM ZbdN ZcdO ZddP ZedQ ZfdR ZgdS ZhdT ZidU ZjdV ZkdW ZldX ZmdY ZndZ Zod[ Zpd\ Zqd] Zrd^ Zsd_ ZtdS )f    N)reduce)generate_control_dev_var_name)is_persistable)coreop_namescopegradient_clipz@PS_STEP_COUNTER@z@LR_DECAY_COUNTER@)cpugpuxpu)sendrecvfetch_barriersend_barrierlookup_tablelookup_table_v2W)r   r   )lookup_table_gradlookup_table_v2_gradr	   )z.batch_sizez
.batch_sumz.batch_square_sumc                     g | ]}|d z   S )@GRAD ).0xs     b/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/ps/utils/public.py
<listcomp>r   7   s    ;;;qq7{;;;    c                    t          j        |          }|                    t           j                   t          j        | ddd          }|                    t           j                   t          j        d          }|                    |           t          j                    }|                    t           j	                   |
                    |           |
                    |           |S )N)levelazUTF-8T)modeencodingdelayzD%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s)logging	getLoggersetLevelWARNINGFileHandlerINFO	FormattersetFormatterStreamHandlerDEBUG
addHandler)log_pathlogging_nameloggerhandler	formatterconsoles         r   logger_configr4   :   s    |,,F
OO'/O***!sWD  G W\"""!N I ####%%GW]###
g
gMr   z	./ps_log/z./ps_usr_print_logps_usr_print_log)r.   r/   c                   &    e Zd ZdZdZdZdZdZdZdS )DistributedModer                  N)	__name__
__module____qualname__SYNCASYNC
HALF_ASYNCGEOFLNUr   r   r   r7   r7   R   s,        DEJ
C	
B	
BBBr   r7   c                       e Zd Zd Zd ZdS )TrainerRuntimeConfigc                    d | _         t          j        dd          }|}|j        d         }|j        s|dk    rt
          j        | _         |j        r|dk    rt
          j        | _         |j        r|dk    rt
          j        | _         |}i | _	        t          j        d|          | j	        d<   t          j        d|          | j	        d<   t          j        d	d          | j	        d
<   t          j        d|          | j	        d<   t          j        dd          | j	        d<   t          j        dd          | j	        d<   t          j        dd          | j	        d<   d S )NCPU_NUM1k_stepsr   $FLAGS_communicator_max_merge_var_numcommunicator_max_merge_var_num"FLAGS_communicator_send_queue_sizecommunicator_send_queue_size*FLAGS_communicator_independent_recv_thread$communicator_independent_recv_thread0FLAGS_communicator_min_send_grad_num_before_recv*communicator_min_send_grad_num_before_recv#FLAGS_communicator_thread_pool_size5communicator_thread_pool_size"FLAGS_communicator_send_wait_timescommunicator_send_wait_times#FLAGS_communicator_is_sgd_optimizercommunicator_is_sgd_optimizer)
r    osgetenva_sync_configsa_syncr7   r@   rA   rC   runtime_configs)selfvalid_strategynum_threadssend_queue_sizerK   s        r   __init__zTrainerRuntimeConfig.__init__\   s   	i	3//% /	:$ 	-A',DI  	.W\\'-DI  	&Wq[['+DI%O!AC2OB
 B
=> @By0/@
 @
;< IBCHH 	CD IBK  	IJ
 AC	13A
 A
<= @By0#@
 @
;< AC	13A
 A
<===r   c                     g }t          j        dd          }d} j         j        t          j        k    r j                                        }d}n\ j        t          j        k    s j        t          j        k    rd}g d}n+ j        t          j	        k    rd}g d}nt          d	           j        t          j        k    s j        t          j        k    rr j        d
         } j        d         }||k    r&t          d| d| d| d| d	           | j        d
<   ||k    r&t          d| d| d| d| d	           | j        d<    fd|D             S )NrI   rJ    asynczsync or half_async)rM   rX   rV   rO   rC   )rV   rX   rM   rO   zUnsupported ModerM   rO   zWARNING: In zo mode, communicator_max_merge_var_num must be equal to CPU_NUM. But received, communicator_max_merge_var_num = z, CPU_NUM = z3. communicator_max_merge_var_num will be forced to .zk mode, communicator_send_queue_size must be equal to CPU_NUM. But received, communicator_send_queue_size = z1. communicator_send_queue_size will be forced to c                 F    i | ]}|t          j        |                   S r   )strr_   )r   keyr`   s     r   
<dictcomp>z?TrainerRuntimeConfig.get_communicator_flags.<locals>.<dictcomp>   s+    IIIS-c233IIIr   )r[   r\   r    r7   rA   r_   keysr@   rB   rC   
ValueErrorprint)r`   	need_keysrb   mode_strmax_merge_var_numrc   s   `     r   get_communicator_flagsz+TrainerRuntimeConfig.get_communicator_flags   s#   	i	3//9	_-B B B,1133IHHI---yO666+H  II Y/---H  II /000 I---yO666 $ 40! #2.O !K//f8 f f8If f #f f Xcf f f     $%EF +--d8 d d6Ed d #d d Vad d d     $%CD JIIIyIIIIr   N)r=   r>   r?   rd   rs   r   r   r   rG   rG   [   s8        '
 '
 '
R<J <J <J <J <Jr   rG   c                 d   g }t          |                                 j                  D ]\  }}t          |                    t
                              }|t          t                    k    s-|t          t                    t          t                    z  k    r|                    |           |S N)		enumerateglobal_blockopsintattrRPC_OP_ROLE_ATTR_NAMELR_SCHED_OP_ROLE_ATTR_VALUEOPT_OP_ROLE_ATTR_VALUEappend)programlr_opsindexoprole_ids        r   
get_lr_opsr      s    Fw33559::  	rbgg34455c56666'S'F
 F
&''F( ;( ;( MM"Mr   c                    |                                  }g }|j        D ]}t          |          rt          |          dk    r|                    d          d         |vrBt
          |                                v rYt          |                    t
                    v r8|	                    dt          t          j        j        j                             |                    |           |S )Nr   Paramop_role)rw   rx   _is_opt_role_opleninputOP_NAME_SCOPE	all_attrsCLIP_OP_NAME_SCOPErz   	_set_attrry   r   op_proto_and_checker_makerOpRoleBackwardr~   )_programremote_sparseblockopt_opsr   s        r   get_optimize_opsr      s    !!##EGi  2 	M""Q&&HHW%%a(== //&"''-*@*@@@7>GHH   NN2Nr   c                     |                                  }g }|j        D ]"}|j        dk    r|                    |           #|S )N	data_normrw   rx   typer~   )r   r   r   r   s       r   get_datanorm_opsr      sN    !!##EGi  7k!!NN2Nr   c                  *   t          t          j        dd                    } d}d}d}t          j        d          rQt          j        d          }|                    d          |          }t	          |                    d                    }| |||dS )NPADDLE_TRAINER_ID0rf   r   PADDLE_TRAINER_ENDPOINTS,)
trainer_idnum_trainerscurrent_endpointtrainer_endpoints)ry   r[   r\   splitr   )r   r   r   r   s       r   get_dist_envr      s    RY2C8899JL	y+,, 9I&@AA,22377
C,2237788 !$,.	  r   c                 t    	 |                                  S # t          $ r |                                 cY S w xY wru   )_role_id	Exceptionr   
role_makers    r   get_role_idr     sL    $""$$$ $ $ $!!#####$    77c                     	 |                                  t          |                    S # t          $ r* |                                 t          |                    cY S w xY wru   )_get_pserver_endpointsr   r   get_pserver_endpointsr   s    r   get_ps_endpointr   	  sk    K0022;z3J3JKK K K K//11+j2I2IJJJJKs   &) 1AAc                 t    	 |                                  S # t          $ r |                                 cY S w xY wru   )r   r   r   r   s    r   get_ps_endpointsr     L    200222 2 2 2//111112r   c                 *    |                                  S ru   )_get_heter_worker_endpointr   s    r   get_heter_worker_endpointr     s    00222r   c                 *    |                                  S ru   )_get_trainer_endpointr   s    r   get_trainer_endpointr     s    ++---r   c                 *    |                                  S ru   )_get_trainer_endpointsr   s    r   get_trainer_endpointsr     s    ,,...r   c                 t    	 |                                  S # t          $ r |                                 cY S w xY wru   )_get_previous_trainersr   get_previous_trainersr   s    r   get_previous_stage_trainersr   #  r   r   c                     | j         t          v r|                     d          du rdS | j         dk    r|                     d          du rdS dS )Nis_distributedTdistributed_lookup_tableFr   SPARSE_OP_LISTrz   r   s    r   is_distributed_sparse_opr   *  s]    	w.  RWW-=%>%>$%F%Ft 	---GG$%%--t5r   c                 8    |                      d          d         S )Nr   r   )r   r   s    r   get_sparse_tablenamer   7  s    88C==r   c                     | j         t          v r0|                     d          du r|                     d          du rdS | j         dk    r|                     d          du rdS dS )N	is_sparseTr   Fr   r   r   s    r   is_sparse_opr   ;  su    
>!!GGK  D((GG$%%..t 	---GG$%%..t5r   c                    t                      }| D ]}|rN|                                j        D ]3}t          |          r"|                    t          |                     4R|                                j        D ]3}t          |          r"|                    t          |                     4t          |          S ru   )setrw   rx   r   addr   r   list)programsr   
tablenamesr   r   s        r   get_sparse_tablenamesr   L  s    J = = 	=**,,0 = =+B// =NN#7#;#;<<<= **,,0 = =## =NN#7#;#;<<<= 
r   c                 t    	 |                                  S # t          $ r |                                 cY S w xY wru   )_worker_numr   
worker_numr   s    r   get_trainersr   Z  sL    '%%''' ' ' '$$&&&&&'r   Fc                 P   t          |          dk     r|S |sg }g }|D ]f}d}	|d         }
|
j        j        }t          D ]}|                    |          rd}	|	r|                    |           Q|                    |           gg }d}|D ]l}|d         }
|                    |
j        j                   |                                 j        |
j        j                 }|t          d |j	        d          z  }mdt          |          z   }d}ddlm}  |||gdg|g|||dd|ddt          |           g           }|||<   |dz  }t          |          dk    r|S g }d}|D ]l}|d         }
|                    |
j        j                   |                                 j        |
j        j                 }|t          d	 |j	        d          z  }md
t          |          z   }d}ddlm}  |||gdg|g|||dd|ddt          |           g           }|||<   |dz  }n|D ]}|d         }
|
j        j        }|                                 j        |         }t          d |j	        d          }|}d}ddlm}  |||gdg|g|g||dd|ddt          |           g           }|||<   |dz  }|S )Nr8   FTr   c                     | |z  S ru   r   r   ys     r   <lambda>z(get_dense_send_context.<locals>.<lambda>  
    QU r   zDense@GRAD_CommContext127.0.0.1:6071c                     | |z  S ru   r   r   s     r   r   z(get_dense_send_context.<locals>.<lambda>  r   r   zDataNorm@GRAD_c                     | |z  S ru   r   r   s     r   r   z(get_dense_send_context.<locals>.<lambda>  
    AE r   )r   
merged_varnameDATA_NORM_GRAD_NAMEendswithr~   rw   varsr   shaperj   paddle.base.corer   id)r   send_ctxidxmerged_dense_pairsr   split_dense_tabledense_pairsdata_norm_pairsmergedis_data_normgradvarnamer   origin_varnames	var_numelvar	grad_name	aggregater   	dense_ctxdata_norm_ctxorigin_varnames                         r   get_dense_send_contextr   a  s    ""
 n( 
	+ 
	+F L!9Do*G+ ( (##D)) (#'L +&&v....""6**** 	! 	B 	BF!9D""4?#7888&&((-do.BCC 2 2CIqAAAII!CHH,		 	100000KKKwKK
 
	  (q1$$J 	% 	B 	BF!9D""4?#7888&&((-do.BCC 2 2CIqAAAII$s3xx/		 	100000#KKwKK
 
  ,q( 	 	F!9D!_1N&&((-n=C1139a@@I&II444444#!" 7 I  #,HY1HCCJr   c                    | d         t           j        k    rt          d| d          d          i }t          | d                   }| d         }d}t	          |d          }t          |          D ]\  }}| d         |         }|D ]}	|	\  }
}|j        j        }|
j        j        }|| d	         v r*||v rdnd
}|                                j	        |j        j                 }t          d |j        dd          d          }ddlm} t          d||            |||gdg|g|g|dd||d
d
t          |          g           }|dz  }|||                                <   ׌t#          |          dk    rt          d          t#          | d                   dk    r&| d         rt%          || d                   \  }}|||<   |S )Nps_modez	ps mode: z) not matched get_geo_trainer_send_contextr   origin_main_programsr   Tmerged_sparse_pairsr   Fc                     | |z  S ru   r   r   s     r   r   z.get_geo_trainer_send_context.<locals>.<lambda>  r   r   r8   r   z(public get_the_geo_send_context sparse: r   z-GeoSGD require sparse parameters in your net.tensor_table	is_worker)r7   rC   rn   r   r   rv   r   r   rw   r   r   r   r   r   ro   r   var_namer   	_step_ctx)attrsr   r   origin_programsr   distributed_varnamesir   r  r   paramr   r   
param_namer   r   r   r   
sparse_ctxr   ctxs                        r   get_geo_trainer_send_contextr    sC   Y?...Si(SSS
 
 	
 HU<011J23O
C0$GG00 $9 $9
7#$9:1=) "	9 "	9F KE4,I).JU?333 #&:::  &&((-do.BCC1139QRR=!DDI444444:Iy   %!"7 J  1HC.8HZ((**++E"	9H 8}}HIII
5 !!A%%%*<%c5#677	cOr   c                     t           }t          |          }t          |          }dgt          |          z  }|gt          |          z  }ddlm}  ||||||g|ddd| dddg           }||fS )Nr8   r   r   TF)STEP_COUNTERr   r   r   r   r   )	r   r   r   r   	endpointssectionsnamesr   r  s	            r   r  r    s    DZ((J ,,IsS^^#HFS^^#E,,,,,,
+	

 C  9r   c                 2   |dg}i }t          | d                   }| d         }t          d|            d}t          |d          }t          |          D ]G\  }}	| d         |         }
|
D ]/}|\  }}|j        j        }|j        j        }g }|| d         v r|                    |           g }t          t          |                    D ]}|                    | d	|            ||v rdnd
}|		                                j
        |j        j                 }t          |j                  }|rdn|d         |d<   ||v rddlm} t          d|||            ||||||g|dd||d
d
t          |	          |          }|dz  }|||                                <   1It          |          D ]'\  }}	| d         |         }t#          |	|||||          }(t          | d                   dk    r&| d         rt%          || d                   \  }}|||<   |S )Nr   r   r  zis_heter_ps_mode? r   Tr  r   .blockFr   z(public get_the_one_send_context sparse: r8   r   r  r  )r   ro   r   rv   r   r   r~   ranger   rw   r   r   r   r   r   r   r  r   r  )r	  r   ep_listr   r   r
  r   r  r  r   r  r   r  r   r   r  remote_sparse_idssplited_varnamer   r   r   r   r  r   r   r  s                             r   get_the_one_send_contextr  3  s   #$HU<011J23O	
20
2
2333
C0$GG00 49 49
7#$9:1=) 2	9 2	9F KE4,I).J "U?333!((--- O3w<<(( A A&&*'?'?A'?'?@@@@ #&:::  &&((-do.BCCOOE*8qqaE!HH$$444444:	   %7! J" 1HC.8HZ((**++e2	9h  00 	
 	

7"#78;$
 
 5 !!A%%%*<%c5#677	cOr   c           
        * |t           vrt          d| dt                      dd}dd}d }|                                 }|                                 *	 i }t	          *j                  }t          |          }t          |dz
  dd          D ]}	t	          *j                  }||	         }
d	|
j        v r|
j        	                    d	          d
         }|t                                          v r"|
                    d          du r
|
                    t          |                   d
         }||v rt	          *j                  }|||                  }|j        d
         *fd|j        D             i}|j        d
         *fd|j        D             i}*                    |	dz   |j        |||                                           *                    ||         dz              |                    |           |D ]}||xx         dz  cc<   |dk    rG	 g }|
j        D ]T}|
                    |          D ]<}|dk    r	d|v r|                    |	                    d          d
                    =U|D ]}||v r	 t	          *j                  }|||                  }|j        d
         *fd|j        D             i}|j        d
         *fd|j        D             i}*                    |	dz   |j        |||                                           *                    ||         dz              |                    |           |D ]}||xx         dz  cc<   |
j        dk    r|
                    d          d
         }d|v r|	                    d          d
         }||	dz
           }d	|j        v r]|j        	                    d	          d
         }|t                                          v rx|                    d          du ra|                    t          |                   d
         }||k    r.|
                    d          |                    d          k    r|	||<   |dk    rg }|j        D ];}|                    |          D ]#}|dk    r	d|v r|                    |           $<g }|
j        D ];}|
                    |          D ]#}|dk    r	d|v r|                    |           $<d}|D ]
}||v rd} n|r|	||<   |	||<   |                                 }|                                 *g }|i i}i }d
}g }g } |}!d}"*j        D ]U}
 ||
|!|          rd}"t          |           dk    r'| ||         |<   |                    |            g } |dz  } ||
|!|          r#|
                    d          }! ||
||           ||d
                             d          }#|||#         |<   |                    |           |dz  }g }|
                    d          }! ||
||           |"r]|d
                             d          }#|||#         |<   |                    |           |dz  }g }|}!d}"|                     |
           @|                     |
           W| g k    r | ||         |<   |                    |            |g k    r;|d
                             d          }#|||#         |<   |                    |           t          |          d
k    rt1          j        d           d
}$d
}%|                                D ]H}&||&         }'|%t          |'          z  }%|'                                D ]\  }(})|$t          |)          z  }$It7          dt          *j                   d|$ d|% d           ||||fS ) NzGiven device z is not in device list r	   c                     t          t                    }|                    |           |                     d          }| j        }||v rdS |t
          v r||k    rdS |||k    r|                     d|           dS dS N	op_deviceTF)r   DEVICE_LISTremoverz   r   COMMUNICATE_OPS_TYPEr   )r   current_heter_devicedefault_deviceheter_devicesr"  op_types         r   _is_heter_opz$find_heter_ops.<locals>._is_heter_op  s    [))^,,,GGK((	'%%4+++$66
 4)~"="=LLn5555ur   c                 P    |                      d          }||k    rdS ||k    rdS dS r!  )rz   )r   
pre_devicer'  r"  s       r   _is_same_devicez'find_heter_ops.<locals>._is_same_device  s8    GGK((	
""4''4ur   c                 l    |                      d          }||vri ||<   |                    |            d S )Nr"  )rz   r~   )r   current_heter_block_ops	heter_opsr"  s       r   _append_heter_opz(find_heter_ops.<locals>._append_heter_op  sB    GGK((	I%%#%Ii &&r*****r   r8   r  _gradr   remote_prefetchTc                 *    g | ]}j         |         S r   r   r   r   r   s     r   r   z"find_heter_ops.<locals>.<listcomp>  s0     0 0 0 % "Ju-0 0 0r   c                 *    g | ]}j         |         S r   r5  r   outputr   s     r   r   z"find_heter_ops.<locals>.<listcomp>  s0     1 1 1 & "Jv.1 1 1r   r   r   inputsoutputsr	  elementwise_mul@EMPTY@lod_tensor_blocking_queuer   c                 *    g | ]}j         |         S r   r5  r6  s     r   r   z"find_heter_ops.<locals>.<listcomp>  s0     4 4 4$) !&
5 14 4 4r   c                 *    g | ]}j         |         S r   r5  r8  s     r   r   z"find_heter_ops.<locals>.<listcomp>  s0     5 5 5$* !&
6 25 5 5r   sumOutr"  FzsNo heterogeneous OP was found in your program ,  please using static.device_guard() to run OPs on different device.z
There are z( OPs in your main_program, and contains z heter-OPs which is made up of z heter-blocks.r	   )r#  rn   clonerw   r   rx   r   r  r   r   SPARSE_OP_TYPE_DICTrm   rz   r   input_namesinput_arg_namesoutput_namesoutput_arg_names
_insert_opr   
_remove_oppopr9  r~   warningswarnitemsro   )+r   r'  r*  r-  r1  origin_programvar2idxop_listop_sizer  r   forward_op_typer  sum_opsum_op_inputssum_op_outputsvar_output_vars_no_gradrk   r   no_grad_varr   
origin_varpre_opoutput_vars
input_varsis_matchprogram_block_opsdefault_opsr0  block_indexr/  current_default_block_opsr&  is_heterr"  total_heter_opsheter_blocksdeviceheter_block_dict_heter_blockr   s+                                             @r   find_heter_opsrl    sV
   [((PNPP;PP
 
 	
   (   + + + ]]__N  ""E G59ooG'llG7Q;B'' A0 A0uy//QZbg gmmG44Q7O#6#;#;#=#===GG-..$66XX&9/&JKKAN
(("59ooG$WZ%89F*1- 0 0 0 0)/)?0 0 0%M +A. 1 1 1 1*0*A1 1 1&N $$!e#[, .$..00 %    $$WZ%81%<===KK
+++ ' + +* $555 ')#? N NC#%99S>> N N"i//$6'AA$+227==3I3I!3LMMMMN $7 / /K"g-- #'uy//!()=!>".q1 4 4 4 4-3-C4 4 4) #/2 5 5 5 5.4.E5 5 5* (("#a%!'#0$2"("2"2"4"4 )    (()=)ABBBK000$+ / /D#DMMMQ.MMMMw%ii&&q)c>>!$7!3!3A!6J$QU^F&+--*0+*;*;G*D*DQ*G+/B/G/G/I/III &,= > >$ F F)/ 3O D* **!J  *Z77BGG += =!'[!9!9=: =: !)67
 3 3,0AAA*,K'-': @ @/5}}S/A/A !@ !@G'.)';';(0'Bg'M'M(0$/$6$6w$?$?$?$?!@ *,J')~ ? ?/1xx}} !? !?G'.)';';(0'Bg'M'M(0$.$5$5g$>$>$>$>!? (-H+6 * *#*j#8#8/3H$)E $9  ( 8 (67
 3./
+]]__N  ""E!2&KIK  ")Hi (1 (1<0.AA '	1H ,--11- N+K8 "(()BCCC,.)q r#7HH I')ww{';';$  %<iHHHH 4A6;;KHH	4K	)$[1!(()@AAAq *,'')ww{';';$  %<iHHHH 	1/277DDI0GIi -$$%<===1K&(##1 H%,,R0000 &,,R0000 B&&3LN#K0  !:;;;"$$+A.33K@@	,C	)[)  !8999
9~~R	
 	
 	

 OL.."" 0 0$V,,---.4466 	0 	0NA{s;///OO	0	 	ZS^^  	Z  	Z_  	Z  	Z  ~J  	Z  	Z  	Z   9k3DDDr   c                    	 t          |           }g }|dz  dk    s
J d            t          d|dz            D ]D}d| |         i}|                    d| |dz
  |z
           i           |                    |           Eg g d}| |dz           D ]M}d|j        vr'|j        d	k    s|d                             |           2|d                             |           N|                    |           |S )
z
    before analyzing the input & output of each block in program_block_list, we should
    union the forward op and corresponding gradient op to eliminate the unnecessary variable
    transmit
    r9   r   z2the length of program_block_ops_list should be oddforwardbackwardr8   )rn  ro  r2  rB  )r   r  updater~   r   )program_block_ops_listblock_lengthunion_program_block_ops_listr  block_op_listr   s         r   union_forward_gradient_opru    sH    -..L#% !q   < !   1la'(( ; ;"$:1$=>/q0@10DEF	
 	
 	
 	%++M:::: "33M$\Q%67 1 1"'!!27e+;+;)$++B////*%,,R0000 ''666''r   c                 l    t          | |          }t          | |||          }t          | ||          }|S ru   )find_entrance_exit_privateentrance_exit_checkdelete_block_useless_exit)r   rq  r0  block_var_details       r   find_block_jointsr{    sX    1'  +')99  1')9  r   c                 p   g }g }|D ]t}t          |                                 j        |          }|t          |          z  }t	          |                                 j        |          }|t          |          z  }ut          t          |                    }t          t          |                    }||fS ru   )_get_input_map_from_oprw   r   get_varlist_from_op_map_get_output_map_from_opr   r   )r   ops_listinput_var_listoutput_var_listr   r;  r<  s          r   find_ops_list_input_outputr    s    NO < <'(<(<(>(>(CRHH1&999)'*>*>*@*@*ErJJ27;;;#n--..N3//00O?**r   c           	         g }g }t          |          D ]\  }}t          | |d                   \  }}t          | |          t          | |          z   }t          t	          |          t	          |          z            }t          t	          |          t	          |          z
            }	t          t	          |          t	          |          z
            }
d|	|
||di}t          | |d                   \  }}t          | |          t          | |          z   }t          t	          |          t	          |          z            }t          t	          |          t	          |          z
            }t          t	          |          t	          |          z
            }|                    d||||di           |                    |           |S )Nrn  )entranceexitprivatepersistablesro  )rv   r  screen_persistablesr   r   rp  r~   )r   rq  rz  r  r   rt  block_inputblock_outputblock_private_varsblock_entrance
block_exitdetailbp_block_inputbp_block_outputbp_persistablesbp_block_private_varsbp_block_entrancebp_block_exits                     r   rw  rw    s
   L )*@ A A ,( ,(}$>]9-%
 %
!\ +[
 
667 "#k"2"2S5F5F"FGGc+..5G1H1HHII#l++c2D.E.EEFF
*"- ,	 
 +E]:.+
 +
' .^
 
99: !%S%8%83;O;O%O P P #&;"<"<<
 
 S11C8M4N4NNOO 1)4$3	 		
 		
 		
 	''''r   c                    t          t          |          dz
  dd          D ]}|dz
  dk     r n||dz
           d         d         }|                                 ||         d         d         }||         d         d         }||         d         d         ||         d         d         z   ||         d         d         z   }|D ]}	d	|	vr|	|vr|                    |	            |                                 ||k    rt	          t          |          t          |          z            }
t	          t          |          t          |
          z
            }||dz
           d         d         }||dz
           d         d         }|D ]M}	|	|vr|	|vr|                    |	           |                    |	           |	|vr|                    |	           Nt          dt          |          dz
  d          D ]v}||dz            d         d         }|                                 ||         d         d         }|                                 ||k    r]t	          t          |          t          |          z            }
t	          t          |          t          |
          z
            }g }|D ]}	d	|	vr|                    |	           t	          t          |                              t          |                              }||dz            d         d         }||dz            d         d         }|D ]4}	|	|vr|	|vr|                    |	           |                    |	           5x|S )
Nr8   r  r   rn  r  r  ro  r  r   )r  r   sortr~   r   r   
difference)r   rq  rz  r0  r   previous_block_exitcurrent_block_entrancebackward_entranceforward_allr   
exist_varsneed_add_varsprevious_block_privateprevious_block_entranceneed_ignore_varss                  r   rx  rx    s    s+,,q0"b99 -3 -319q==E.uqy9)DVL  """!1%!8!CJ!O,U3J?
K U#I.z:u%i0;<u%i089 	 % 	3 	3Cc!!c&<&<&--c222##%%%"888#$$s+A'B'BB
 

 S!7883z??JKK
 "2%!)!<Y!G"
 #3519"=i"H#
 ! 	3 	3C111666'..s333&&s+++000&--c222	3 q#.//!3Q77  ,  ,.uqy9*EfM  """!1%!8!DZ!P##%%%"888#$$s+A'B'BB
 

 S!7883z??JKK  	- 	-Cc!! '',,,))#.>*?*?@@
 
 "2%!)!<Z!H"
 #3519"=j"I#
 ! 	, 	,C111666'..s333&&s++++	, r   c                 f   t          t          |                    D ]}|t          |          dz
  k    r nf||         d         d         }||dz            d         d         }g }|D ]}||vr|                    |           |D ]}|                    |           t          t          |          dz
  dd          D ]r}|dz
  dk     r nf||         d         d         }||dz
           d         d         }g }|D ]}||vr|                    |           |D ]}|                    |           s|S )Nr8   rn  r  r  r  r   ro  )r  r   r~   r$  )r   rq  rz  r   current_block_exitnext_block_entranceneed_delete_varr   s           r   ry  ry  R  s    s+,,-- + +C())A---E-e4Y?G.uqy9)DZP% 	, 	,C---&&s+++" 	+ 	+C%%c****	+ s+,,q0"b99 + +19q==E-e4Z@H.uqy9*E
 % 	, 	,C---&&s+++" 	+ 	+C%%c****	+ r   rn  c                 Z   g }g }|dk    rd|dz
   d| d}nd|dz    d| d}|                                  |D ]j}|                                 j        |         }|j        }	dt	          d |	d          z  }
|                    |
           |                    | d	           k|||d
}|S )Nrn  forward_joint_r8   rj  z@Heterbackward_joint_r  c                     | |z  S ru   r   r   s     r   r   z*get_communicate_var_info.<locals>.<lambda>  s
    A r   z.input_reshape@Heter)input_var_reshape_diminput_var_reshape_nameblock_input_var_name)r  rw   r   r   r   r~   )r   rc  entrance_var_listr   r  r  r  r   r   r   recv_var_diminfos               r   get_communicate_var_infor  t  s    yB[1_BB{BBB 	
 DkAoCCCCC 	  " E E""$$)$/	F#5#5ua@@@$$\222%%&C&C&CDDDD "7"8 4 D Kr   c                 &   | D ]}||                                 j        vrp||j        vrg|                                 j        |         }|j        r*|                                                     |d           v|                    |d           d S )NFforce_persistable)rw   r   persistable_clone_variable)var_name_listrQ  r   r   r  r   s         r   add_vars_by_var_listr    s    ! D DG0022777
** --//4X>C D$$&&665 7     %%cU%CCCD Dr   c                 
   t          j                    }|j        D ]g}g }|                    |          D ])}|dk    r	d|v r|                    | |                    *t          |          dk    r|d         ||<   b|||<   h|S )z9Returns a dict from op output name to the vars in varmap.r>  r?  r8   r   )collectionsOrderedDictrI  r9  r~   r   varmapr   iomaprk   r   r   s         r   r  r    s    #%%E  yy~~ 	) 	)G)##*g55KKw((((t99>>aE#JJE#JJLr   c                     g }|                                  D ]^\  }}t          |t                    s|g}t          t	          |                    D ]$}||         }|                    |j                   %_|S ru   )rP  
isinstancer   r  r   r~   r   )var_mapvar_listrk   varlistr  r   s         r   r~  r~    s    H & &W'4(( 	 iGs7||$$ 	& 	&A!*COOCH%%%%	& Or   c                 
   t          j                    }|j        D ]g}g }|                    |          D ])}|dk    r	d|v r|                    | |                    *t          |          dk    r|d         ||<   b|||<   h|S )z8Returns a dict from op input name to the vars in varmap.r>  r?  r8   r   )r  r  rG  r   r~   r   r  s         r   r}  r}    s    #%%E~  xx}} 	) 	)G)##*g55KKw((((t99>>aE#JJE#JJLr   c                    g }|D ]}d|v r[d|                     d          d         k    r&|                     d          d         }|                                 j        |         }n|                                 j        |         }t          |          r|                    |           |D ]}|                    |           |S )Nr   GRAD@r  r   )r   rw   r   r   r~   r$  )r   r  need_remover  origin_var_namer   s         r   r  r    s    K 
) 
)h,,R000&nnW55a8O&&((-o>CC&&((-h7C# 	)x((( " "!!!!r   c                    |                                 j                                        }|                    |j                   t	          ||          }|                                D ]\  }}t          |t                    s|g}|D ]x}|j        |                                  j        vrV|j        |j        vrH|j	        r*|                                  
                    |d           a|
                    |d           yt          |                                 j        |          }	|	                                D ]\  }}t          |t                    s|g}|D ]x}|j        |                                  j        vrV|j        |j        vrH|j	        r*|                                  
                    |d           a|
                    |d           yd|j        vr0|                    |j        ||	|                                          S |j        }
t           j        j        j        }t           j                                        }|j                                        }|                    |
           |                    t.          |           |j                            |          r+|
                    |          }|                    ||           |                                 d S )NFr  r2  r   r;  r<  r	  )rw   r   copyrp  r}  rP  r  r   r   r  r  r  r   	append_opr   descr   r   r   r   kOpDeviceAttrName	copy_fromr   r{   has_attrrz   _sync_with_cpp)r   rQ  r   r   merge_ordereddictr;  rk   r  r   r<  op_descro  device_attr_namenew_op_descr"  s                  r   block_append_opr    s   &3355:??AAUZ(((#$5r::F H HW'4(( 	 iG 
	H 
	HC 4 4 6 6 ;;;HEJ..? H((**::u ;     ))#)GGG
	H &n&A&A&C&C&H"MMG H HW'4(( 	 iG 
	H 
	HC 4 4 6 6 ;;;HEJ..? H((**::u ;     ))#)GGG
	H bg  
 
 	

 '29B:LLNN j**,,g&&&3X>>> 7,-- 	?%566I!!"2I>>>r   c                 t    	 |                                  S # t          $ r |                                 cY S w xY wru   )_get_next_trainersr   get_next_trainersr   s    r   get_next_stage_trainersr  !  sL    .,,... . . .++-----.r   Tc                    |rGt          |          }t          |          }	||         d         d         }
t          | |dz   |
          }nJt          |          }t          |          }	||dz
           d         d         }
t          | |dz
  |
d          }|                    |dd|j        |
d                  id	g id
|rdnddg |
ddg d|d         d|d|	dt          |          d|t          t          i	           |
S )Nrn  r  r8   ro  r  send_and_recvXr   rC  r    send_var_namemicrobatch_idrecv_var_namemessage_namer  next_endpointsprevious_endpointsr   r"  r:  )r  r   r  rK  r   r   r{   RPC_OP_ROLE_ATTR_VALUE)rQ  r   rk  stage_idfirst_op_indexrz  rh  
is_forwardnext_heter_worker_endpointsprevious_heter_worker_endpointsentrance_var	comm_infos               r   insert_communicate_opr  (  sW     
&=j&I&I#*E+
 +
' (1)<ZH,HqL,
 
		
 '>j&I&I#*E+
 +
' (15jA&I,HqL,

 
	 [%l1o67;II=|=_=RI&<=9 "A+j11!#9

    $ r   c                    i }i }|rt          | |          }t          |                                          D ]\  }\  }}|                                r|                                r2|                                }	g }
|	D ]%}| d         |         }|
                    |           &|
||                                <   nt          | dd           }t          |                                          D ]r\  }\  }}|                                s|                                }	g }
|	D ]%}| d         |         }|
                    |           &|
||                                <   s|S )N)r   grad_name_to_param_nameF)r   r  )r  rv   rP  r   is_tensor_tabler   r~   table_id)contextis_denser   recv_id_mapsr  r   r   r   r  origin_grad_varnamesparam_namesgrad_varnamer  s                r   get_the_one_recv_contextr  [  s   L  7+'8
 
 
 !*(..*:*: ; ; 	7 	7C$}} ""$$ #&#6#6#8#8 K 4 / /$%>?M
"":....+6L((	7 ,ud
 
 
 !*(..*:*: ; ; 
	7 
	7C$==?? #&#6#6#8#8 K 4 / /$%>?M
"":....+6L((r   c                 (   d}d}d}|                      d          }|dk    r| |dz   d          }nt          |           }|                      d          }|dk    r| |dz   |         }nt          |           }| dt          ||                   }|||fS )Nrf   z	.trainer_r   r8   r  )findr   min)r   orig_var_nametrainer_part
block_parttrainer_idxrc  s         r   _get_varname_partsr    s    MLJ,,{++Ka{Q001'll,,x((Ka[1_{:;

'llAK = ==>M*l22r   r9   r;      r8   c                 ^    t          d | j        d          }|t          | j                 z  }|S )Nc                     | |z  S ru   r   r   s     r   r   z"get_var_mem_size.<locals>.<lambda>  s
    Q r   r8   )r   r   dtype_to_sizedtype)r   m_sizes     r   get_var_mem_sizer    s/    &&	155F
mCI&&FMr   c                       e Zd Zd ZdS )MergedVariablec                 0    || _         || _        || _        d S ru   )r   ordered_varsoffsets)r`   r   orderedr  s       r   rd   zMergedVariable.__init__  s     #r   N)r=   r>   r?   rd   r   r   r   r  r    s#            r   r  c                    | d         }i }i }g | d<   g | d<   g | d<   g | d<   g | d<   i | d<   |D ]}t          |          \  }}g }g }g }	g }
g }|D ]\  }}|                    ||f           |D ]\  }}|                    ||f           |D ][}|\  }}t          ||gdg          }t          ||gdg          }|                    ||f           |
                    ||f           \|D ][}|\  }}t          ||gdg          }t          ||gdg          }|                    ||f           |	                    ||f           \|D ];}|\  }}|j        | d         |j        j        <   |j        | d         |j        j        <   <g }|                    |           |                    |           |D ]#\  }}|j        ||j        <   |j        ||j        <   $| d                             |           | d                             |           | d                             |	           | d                             |
           || d	<   || d
<   d S )Nr  origin_sparse_pairsorigin_dense_pairsr  r   merged_variables_pairsmerged_variable_mapr   param_name_to_grad_namer  )get_param_gradsr~   r  r   r   extend)r  r
  r  r  rQ  sparse_pairsr   origin_for_sparseorigin_for_denser  r   r  r  r   
dense_pairm_paramm_gradsparse_pairr   param_mergess                       r   build_var_distributedr    s5   45O  %'G!"$&G !%'G!"$&G !(*G$%%'G!") 8A 8A$3N$C$C!k  !#' 	4 	4KE4$$eT]3333& 	3 	3KE4##UDM2222* 	9 	9J$KE4$UUGaS99G#D4&1#66F"))7F*;<<<%%w&78888 - 	: 	:K%KE4$UUGaS99G#D4&1#66F"))7F*;<<<&&'89999 - 	 	F$OGV" )*7+=+BC ! )*6+<+ABB -...,---' 	< 	<KE426)#EJ/16#DI..%&--.?@@@$%,,-=>>>%&--.ABBB$%,,-?@@@@)@G%&)@G%& r   c                    t           j        }t           j        j        j        }|                                | j        v rLt          |                                 |                                                   t          |          k    rdS dS )NTF)r   r   r   OptimizekOpRoleAttrName
attr_namesry   r   )r   op_makeroptimize_roles      r   r   r     s|     .H3:CM!!R]22s
x//1128 8	]		8 8 t5r   c                 P      fd} fd} |            } ||          \  }}||fS )Nc                 J                                    }g }g }t                      }                                 j        }t          t          j        j        j                  }|j        D ]4}t          |          r!t          |                                v r8t          |                    t                    v r|                    d|           f|                    t                     s|                    t                     r|                    t                     d         }|                    t                     d         }	||vrU|                    |           ||         ||	         f}
|| v r|                    |
           |                    |
           6||fS )Nr   r   r8   )rw   r   r   ry   r   r   r   r   rx   r   r   r   r   rz   r   r  OP_ROLE_VAR_ATTR_NAMEr   r~   )sparse_varnamesr   dense_param_gradssparse_param_gradsoptimize_paramsorigin_var_dictr   r   r  r   
param_gradrQ  s              r   _get_params_gradsz*get_param_grads.<locals>._get_params_grads  s   ++--%%(5577<d5<EFF) 	A 	ABr"" A "R\\^^33*bggm.D.DDDLLG444{{#899 77011 A!#)>!?!?!BJ "(= > >q AI!88'++J777+J7+I6&

 &88.55jAAAA-44Z@@@!#444r   c                  f   g }                                  j        D ]y}|j        t                                          v rW|                    d          du r@|                    t          |j                           d         }|                     |           zt          t          |                     S )Nr3  Tr   )
rw   rx   r   rF  rm   rz   r   r~   r   r   )varnamesr   r  rQ  s      r   _get_sparse_varnamesz-get_param_grads.<locals>._get_sparse_varnames5  s     --//3 	, 	,B.335555GG-..$66XX&9"'&BCCAF

+++CMM"""r   r   )rQ  r-  r0  r'  r)  r(  s   `     r   r  r    si    "5 "5 "5 "5 "5H
# 
# 
# 
# 
# +*,,O,=,=o,N,N))000r   c                     |D ]e}	 t          | j                                      |          }|                     |           @# t          $ r}t          |           Y d }~^d }~ww xY wd S ru   )r   rx   r   rL  r   ro   )r   rx   r   r   es        r   
delete_opsr3  G  s      	uy//''++CS!!!! 	 	 	!HHHHHHHH		 s   <A
A&A!!A&c                     g }|                                  j        D ]"}|j        dk    r|                    |           #|S )Nr   r   )r   send_op_listr   s      r   find_send_opr6  P  sM    L""$$( $ $7f###r   c                    g }g }t          |j        |          }|t          |          z  }t          |j        |          }|t          |          z  }t	          t          |                    }t	          t          |                    }||fS ru   )r}  r   r~  r  r   r   )r   r   r   r  r  r;  r<  s          r   find_op_input_outputr8  X  s    NO#EJ33F-f555N%ej"55G.w777O#n--..N3//00O?**r   c                      fd}g } |            }i }|D ]}d|vrd|                     d          d         k    r'||vr,||         }|                    d          }	|                    d          }
|                    d          }|                    |           |
|vr&i ||
<   g ||
         d	<   |	||
         d<   |||
         d<   ||
         d	                             |           |D ]y}
                    t	                      
          }fd||
         d	         D             }                    dd|id|id||
         d         d|	d|
t          t          i           z|S )Nc                      i } t                    }|D ]2}t                                          |          \  }}|D ]}|| |<   3| S ru   )r6  r8  rw   )send_op_dictr5  r   
input_listrj  r   r   s         r   _get_send_op_dictz&add_send_op.<locals>._get_send_op_dicte  ss    #G,, 	' 	'B0--// MJ " ' '$&S!!'r   r   r  r  r  r   r  send_varnamesr  )r   c                 *    g | ]}j         |         S r   r5  )r   	union_varr   s     r   r   zadd_send_op.<locals>.<listcomp>  s0     
 
 
 Jy!
 
 
r   r   r  rC  r  )r   rz   r~   
create_varr   r  r{   r  )r   r   _varsr=  send_grad_var_listr;  
table_dictpersistable_varsend_opr   r  r>  dummy_outputsend_input_varss   ``            r   add_send_oprI  d  s   	 	 	 	 	 $$&&LJ  A A/))_**3//333,../LL--	<<
++_55!!/222:%%#%Jx /1Jx ,09Jx -4AJx 18Z(//@@@@ 
 
''-J-L-L'MM
 
 
 
'1*=
 
 
 	)L)H!5o!FYH%'=		 	 
	
 
	
 
	
 
	
 r   c                 V    | j                                         }t          |          }|S ru   )r   rm   r   )r   	vars_listvars_name_lists      r   get_vars_name_in_blockrM    s$    
!!I)__Nr   c                    t          t          |                    }g }|                                 j        D ]}t	          | |                                 |          \  }}t          t          |                              t          |                              }t          t          |                              t          |                              }||z  }t          t          t          |                                                                         t          |                              }|D ])}|                                                     |           *|S ru   )	r   r   rw   rx   r8  unionrM  r  _remove_var)	r   
static_varprogram_useful_var_listr   r  r  op_var_listprogram_useless_var_listr   s	            r   delete_trainer_useless_varrU    sU   c*oo&&J ""$$( 
 
*>W))++R+
 +
' 3~..44S5I5IJJKK"&'((..s;/?/?@@#
 #
 z)#"7#7#7#9#9::;;FF'((	
 	
   
 ( 0 0**3////##r   c                    | j         dz
  }|                     |          }t          |          D ]i\  }}|j        dk    rG|                    d          }d}	|D ]*}
|
|                                 j        vr|
|j        vrd}	 n+|	rWt          | |||           j|d         d         d         }t          ||| |           |d         d         d	         }t          ||| |           |S )
Nr8   r   r>  FTr   ro  r  r  )	
num_blocks_create_blockrv   r   rz   rw   r   r  r  )r   rQ  bp_ops_listrz  pre_block_idxrk  rj  r   r>  is_skipr   entrance_vars	exit_varss                r   create_backward_blockr^    s    &*M''66K;'' B B27fGGO44MG(  7#7#7#9#9#>>>{'777"GE bAAAA$Q'
3J?MMMM #J/7ING[IIIr   c                     t           | j        v o@t          |                     t                               t          t          j                  z  S ru   )op_role_attr_namer!  ry   rz   r   r   r   s    r   is_backward_opra    s>    - BGG%&&''#g.>*?*??r   c                     t           | j        v oAt          |                     t                               t          t          j                  k    S ru   )r`  r!  ry   rz   r   Forwardr   s    r   is_forward_oprd    s=    - BGG%&&''3w+?+??r   c                     | j         dk    S )Ndistributed_push_sparse)r   r   s    r   is_push_sparse_oprg    s    7///r   c                     g }t          | j                                                  D ]3}| j        |         }t	          |          r|                    |           4|S ru   )r  r  rT  rx   rg  r~   )r   push_sparse_op_listop_idxr   s       r   #get_distributed_push_sparse_op_listrk    sd    
**,,-- + +YvR   	+&&r***r   c                     g }t          | j                                                  D ]3}| j        |         }t	          |          r|                    |           4|S ru   )r  r  rT  rx   ra  r~   )r   
bp_op_listrj  r   s       r   get_bp_op_listrn    sc    J
**,,-- " "Yv" 	"b!!!r   c                 .   |D ]}	 | j         D ]`}t          |          t          |          k    r>t          | j                                       |          }|                     |            nal# t
          $ r}t          |           Y d }~d }~ww xY wd S ru   )rx   rj   r   r   rL  r   ro   )r   rx   r   	origin_opr   r2  s         r   delete_same_opsrq    s      	"Y  	y>>SWW,,uy////	::C$$S)))E -  	 	 	!HHHHHHHH	 s   A(A//
B9BBc           	      <   d}| j         D ]}|j        D ]s}|j                                        }|j                                        }||z   D ]9}|                    t          |                    st          d| d|           :t|dz  }t          d           d S )Nr   zvar: z% needed by op is not found in block: r8   zprogram checked valid)	blocksrx   r  rH  rJ  _find_var_recursiverj   rn   ro   )r   	block_idxr   r   input_var_namesoutput_var_namesr  s          r   check_programrx    s    I 	 	) 	 	B g5577O!w7799+.>>  00X?? $ZZZyZZ  
 	Q			
!"""""r   c                    t          j        t           j                            |           d           t	          | d          5 }|                    t          |                     d d d            d S # 1 swxY w Y   d S )NT)exist_okzw+)r[   makedirspathdirnameopenwriterj   )filer   fs      r   debug_programr    s    K%%5555	dD		 Q	G                 s   #A44A8;A8c                  6    t          j        d          } | dS dS )NTRAINING_ROLEFT)r[   r\   )	node_roles    r   is_distributed_envr    s!    	/**Iutr   )F)FNrD  )rn  )T)TF)ur  r#   r[   rN  	functoolsr   paddle.base.frameworkr   paddle.distributed.ior   paddle.frameworkr   r   r   r  LEARNING_RATE_DECAY_COUNTERr   kOpRoleVarAttrNamer&  r   r{   r   RPCr  r   r`  LRSchedr|   r  r}   r   ro  r  OP_DEVICE_KEYr#  r%  r   rF  SPARSE_GRAD_OP_TYPE_DICTDEFAULT_DEVICEDATA_NORM_NAMEr   r4   ps_log_root_dirr0   r7   rG   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  rl  ru  r{  r  rw  rx  ry  r  r  r  r~  r}  r  r  r  r  r  r  VarDescVarTypeFP16FP32FP64INT16INT32INT64BOOLUINT8r  r  r  r  r   r  r3  r6  r8  rI  rM  rU  r^  ra  rd  rg  rk  rn  rq  rx  r  r  r   r   r   <module>r     s        				        ? ? ? ? ? ? 0 0 0 0 0 0 ! ! ! ! ! ! $ "2 7JJLL 7GGII 8?C 

)
03CCEE "=DL 8?H *1:/AACC###HHH  "34'*sCC    CCC;;N;;;   $ 	!0B
 
 

       fJ fJ fJ fJ fJ fJ fJ fJR   .0    0    $$ $ $K K K2 2 23 3 3. . ./ / /2 2 2
 
 
    "  ' ' ' y y y yx8 8 8v  6P P P PfDE DE DE DEN( ( (@  + + +0 0 0fS S Sl  F 3<   DD D D  $    $  &7 7 7t. . . 0 0 0 0f# # # #L3 3 3( 	LqLqLqLLLLqL	         T T Tn	 	 	41 41 41n    	+ 	+ 	+4 4 4n  $ $ $,  8    0 0 0    	 	 	# # #      r   