
    x-j7#                        d dl Z d dlZd dlmZmZmZmZmZmZm	Z	m
Z
 ddlmZmZmZmZ ddlmZmZ  ed           G d d	e                      Z ed
           G d de                      Z ed           G d de                      Z ed           G d de                      Z ed           G d de                      Z ed           G d de                      Z ed           G d de                      ZdS )    N)ExponentialDecayInverseTimeDecayLRSchedulerNaturalExpDecay	NoamDecayexponential_decayinverse_time_decay
noam_decay   )get_optimize_opsget_ps_endpointget_role_idget_trainers   )PassBaseregister_passadd_lr_decay_table_passc                   F     e Zd Z fdZd Zd Z	 	 	 	 d	dZd Zd Z xZ	S )
AddLrDecayTablePassc                 H    t                                                       d S Nsuper__init__self	__class__s    h/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/passes/ps_server_pass.pyr   zAddLrDecayTablePass.__init__(           c                     dS NT r   s    r   _check_selfzAddLrDecayTablePass._check_self+       tr    c                     dS r"   r#   r   
other_passs     r   _check_conflictz#AddLrDecayTablePass._check_conflict.   r&   r     Nc                     i }i ||<   |||         d<   |||         d<   |||         d<   |||         d<   |||         d<   ||d<   d S )Nfeed_var_namefetch_var_namestartup_programmain_programtensor_table_classtensor_tabler#   )r   attrsr-   r.   r/   r0   r1   tensor_table_dicts           r   _add_tensor_tablez%AddLrDecayTablePass._add_tensor_table1   s|     +--(<I-(9=K-()9:>M-():;;G-(8 	-()=> !2nr    c                 4   g d}t           j                                        }t           j                                        }d}t          |t                    rpt           j                            ||          5  t          d||j        d          }|j        }t          j
        d| d           d d d            n# 1 swxY w Y   nt          |t                    ryt           j                            ||          5  t          |j        |j        d          }|j        }t          j
        d|j         d           d d d            n# 1 swxY w Y   n?t          |t                    rt           j                            ||          5  t           j        j                            d|j                                                  }|j        }t          j
        d	| d           d d d            n# 1 swxY w Y   nt          |t&                    rot           j                            ||          5  t)          d||j        d          }|j        }t          j
        d
| d           d d d            n# 1 swxY w Y   nt+          d|           |||fS )N)r   r   r   r   r+   g      ?TzPExponentialDecay is set, staircase = True, global learning rate decay step is [ z ], Change decay steps as follow: 
	 strategy = paddle.distributed.fleet.DistributedStrategy() 
 	 strategy.a_sync = True 
	 strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } 
z$NoamDecay is set, warmup steps is [ z ]zONaturalExpDecay is set, staircase = True, global learning rate decay step is [ zPInverseTimeDecay is set, staircase = True, global learning rate decay step is [ zONot supported current LearningRate strategy, please use follow decay strategy: )paddlestaticProgram
isinstancer   program_guardr   gammanameloggingwarningr   r
   d_modelwarmup_stepsr   	optimizerlrget_lrr   r	   
ValueError)r   lr_schedulerlr_decay_stepsscheduler_decaydecay_main_programdecay_startup_programlr_namerC   s           r   _get_lr_scheduler_programz-AddLrDecayTablePass._get_lr_scheduler_programE   s   
 
 
 $]2244 & 5 5 7 7l$455 8	,,"$9    ');T  '\gu \ \ \                 i00 *	,,"$9  	 	   (,*CS  'X<;TXXX  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 o66 	,,"$9    %(88+ &((  '\ft \ \ \                 &677 	,,"$9    ();T  '\gu \ \ \                 sbqss   "#8'AAsJ   87B;;B?B?=A E		EEAG33G7:G747I77I;>I;c                 L   |j         }t          |d         d          sd S t          |d         j        t                    s
J d            t          |d                   }|                     |d         j        |d                   \  }}}|                     |d|||d           d S )Norigin_main_programrF   zmust be LRSchedulerrG   z@LR_DECAY_COUNTER@GlobalStepTable)_attrshasattrr:   rF   r   r   rL   r5   )	r   r0   r/   pass_ctxr3   opslr_decay_main_programlr_decay_startup_programrK   s	            r   _apply_single_implz&AddLrDecayTablePass._apply_single_impl   s    u23^DD 	F'(5{
 
 	! 	! 	! 	! 
 u%:;<<
 **'(5u=M7N
 
		
!$ 	 $!	
 	
 	
 	r    )r+   NNr+   )
__name__
__module____qualname__r   r%   r*   r5   rL   rV   __classcell__r   s   @r   r   r   &   s                 2 2 2 2(FB FB FBP      r    r   add_listen_and_serv_passc                   0     e Zd Z fdZd Zd Zd Z xZS )AddListenAndServPassc                 H    t                                                       d S r   r   r   s    r   r   zAddListenAndServPass.__init__   r   r    c                     dS r"   r#   r$   s    r   r%   z AddListenAndServPass._check_self   r&   r    c                     dS r"   r#   r(   s     r   r*   z$AddListenAndServPass._check_conflict   r&   r    c                 
   |j         }d d d d d t          |d                   t          |d                   t          |d                   |d         dddd}|                                                    ddg ii |           d S )N
role_makerps_mode)grad_to_block_idsparse_grad_to_paramlr_decay_block_iddense_optimize_blockssparse_optimize_blocksendpoint
pserver_idFanindistributed_moderpc_get_thread_numrpc_send_thread_numrpc_prefetch_thread_numlisten_and_servX)typeinputsoutputsr3   )rP   r   r   r   global_block	append_op)r   r0   r/   rR   r3   opts         r   rV   z'AddListenAndServPass._apply_single_impl   s     $$(!%%)&*'l(;<<%eL&9::!%"566 %i 0"$#%')
 
 	!!##--"C9b 	. 	
 	
 	
 	
 	
r    rW   rX   rY   r   r%   r*   rV   rZ   r[   s   @r   r^   r^      se                
 
 
 
 
 
 
r    r^   add_rpc_global_flags_passc                   0     e Zd Z fdZd Zd Zd Z xZS )AddRpcGlobalFlagsPassc                 H    t                                                       d S r   r   r   s    r   r   zAddRpcGlobalFlagsPass.__init__   r   r    c                     dS r"   r#   r$   s    r   r%   z!AddRpcGlobalFlagsPass._check_self   r&   r    c                     dS r"   r#   r(   s     r   r*   z%AddRpcGlobalFlagsPass._check_conflict   r&   r    c                     d S r   r#   r   r0   r/   rR   s       r   rV   z(AddRpcGlobalFlagsPass._apply_single_impl       r    rz   r[   s   @r   r}   r}      e                      r    r}   add_optimizer_passc                   0     e Zd Z fdZd Zd Zd Z xZS )AddOptimizerPassc                 H    t                                                       d S r   r   r   s    r   r   zAddOptimizerPass.__init__   r   r    c                     dS r"   r#   r$   s    r   r%   zAddOptimizerPass._check_self   r&   r    c                     dS r"   r#   r(   s     r   r*   z AddOptimizerPass._check_conflict   r&   r    c                     d S r   r#   r   s       r   rV   z#AddOptimizerPass._apply_single_impl   r   r    rz   r[   s   @r   r   r      r   r    r   add_geo_optimizer_passc                   0     e Zd Z fdZd Zd Zd Z xZS )AddGeoOptimizerPassc                 H    t                                                       d S r   r   r   s    r   r   zAddGeoOptimizerPass.__init__   r   r    c                     dS r"   r#   r$   s    r   r%   zAddGeoOptimizerPass._check_self   r&   r    c                     dS r"   r#   r(   s     r   r*   z#AddGeoOptimizerPass._check_conflict   r&   r    c                     d S r   r#   r   s       r   rV   z&AddGeoOptimizerPass._apply_single_impl   r   r    rz   r[   s   @r   r   r      r   r    r   "build_pserver_startup_program_passc                   0     e Zd Z fdZd Zd Zd Z xZS )BuildPserverStartupProgramPassc                 H    t                                                       d S r   r   r   s    r   r   z'BuildPserverStartupProgramPass.__init__   r   r    c                     dS r"   r#   r$   s    r   r%   z*BuildPserverStartupProgramPass._check_self   r&   r    c                     dS r"   r#   r(   s     r   r*   z.BuildPserverStartupProgramPass._check_conflict   r&   r    c                     d S r   r#   r   s       r   rV   z1BuildPserverStartupProgramPass._apply_single_impl  r   r    rz   r[   s   @r   r   r      r   r    r   delete_unused_in_startup_passc                   0     e Zd Z fdZd Zd Zd Z xZS )DeleteUnusedInStartupPassc                 H    t                                                       d S r   r   r   s    r   r   z"DeleteUnusedInStartupPass.__init__  r   r    c                     dS r"   r#   r$   s    r   r%   z%DeleteUnusedInStartupPass._check_self  r&   r    c                     dS r"   r#   r(   s     r   r*   z)DeleteUnusedInStartupPass._check_conflict  r&   r    c                     d S r   r#   r   s       r   rV   z,DeleteUnusedInStartupPass._apply_single_impl  r   r    rz   r[   s   @r   r   r     r   r    r   )r>   r7   paddle.optimizer.lrr   r   r   r   r   r   r	   r
   ps.utils.publicr   r   r   r   	pass_baser   r   r   r^   r}   r   r   r   r   r#   r    r   <module>r      s    	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	            / . . . . . . . ())    (   *)D )**
 
 
 
 
8 
 
 +*
@ *++    H   ,+ #$$    x   %$ '((    (   )( 344    X   54 .//       0/  r    