
    x-j`                        d dl Z d dlZd dlZd dlmZ d dlZd dlmZ d dlmZm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZm Z   G d de          Z! G d d          Z" G d d          Z#dS )    N)defaultdict)core)not_to_static	to_static)ProgramTranslatorStaticFunction)as_not_paddle_func)Layer)	Parameterglobal_scopeprogram_guard)DEFAULT_AMP_OPTIONSprepare_op_amp_options   )	Converter)TensorDistAttr)get_world_process_group)
get_loggerto_listc                   &    e Zd ZdZ fdZej        j        d             Zd Z	d Z
d Zed             Zd Zd	 Zd
 Zd Zed             Zed             Zed             Zed             Zed             Zed             Zed             Z xZS )
ProxyLayerz
    ProxyLayer implements all logic for converting dygraph model into
    static Program IR. Meanwhile, it provides conventional interfaces for
    auto parallel to visit feed/fetch/loss/metric variables.
    c                    t                                                       || _        || _        || _        d | _        t          t                    | _        t          t                    | _	        t          t                    | _
        t          t                    | _        t          t                    | _        t          t                    | _        dD ]2}t          t          j        t"                    j         d|            3d S )N)_train_eval_predict	call_losscall_metricsz.ProxyLayer.)super__init__inner_layer	loss_funcmetricsmoder   list_input_vars_label_vars_output_vars
_loss_vars_loss_names_metric_varsr	   inspect	getmoduler   __name__)selflayerr!   r"   fn_name	__class__s        n/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/auto_parallel/static/helper.pyr   zProxyLayer.__init__0   s     "	 't,,&t,,'--%d++&t,,'--
 		 		G $Z009PPwPP   		 		    c                     t           j        j                            d          }t           j                            | j        |         |           || j        |<   d S )Nloss)paddleutilsunique_namegenerate_C_opsset_persistable_valuer(   r)   )r.   r#   names      r2   append_loss_to_shadow_outputz'ProxyLayer.append_loss_to_shadow_outputN   sL    |'0088++DOD,A4HHH!%r3   c                 ~   d}|| j         |<   || j        |<    | j        | | j        |<   |                     | j        |          }|                     |          | j        |<   t          j	        j
                            d          d         r|                     |           |                     |          | j        |<   dS )zN
        Train process of inner_layer with forward/loss/metric logic.
        trainFLAGS_enable_pir_apiNr%   r&   r    r'   _prepareoutput_varsr   r(   r6   base	framework	get_flagsr=   r   r*   r.   inputslabelsr#   
new_inputss        r2   r   zProxyLayer._trainT   s    
 !'!' #3$"2F";$ ]]4#3V<<
 $z : :; **+ABB"
 	4 --d333 #'"3"3J"?"?$r3   c                 ~   d}|| j         |<   || j        |<    | j        | | j        |<   |                     | j        |          }|                     |          | j        |<   t          j	        j
                            d          d         r|                     |           |                     |          | j        |<   dS )zQ
        Evaluate process of inner_layer with forward/loss/metric logic.
        evalr@   NrA   rG   s        r2   r   zProxyLayer._evalk   s     !'!' #3$"2F";$ ]]4#3V<<
 $z : :; **+ABB"
 	4 --d333 #'"3"3J"?"?$r3   c                 V    d}|| j         |<   || j        |<    | j        | | j        |<   dS )zD
        Predict process of inner_layer with forward logic.
        predictN)r%   r&   r    r'   )r.   rH   rI   r#   s       r2   r   zProxyLayer._predict   sB    
 !'!' #3$"2F";$r3   c                 @    t          |          t          |          z   S )z}
        Concat outputs and labels as a single list

        NOTE(dev): We use @not_to_static to avoid AST Analysis.
        )r   )r.   outputsrI   s      r2   rB   zProxyLayer._prepare   s     w'&//11r3   c                 ,    g }| j         
 | j         | }|S )z
        Apply Loss Function on outputs and labels.

        Args:
            inputs: List[Variable]

        Returns: List[Variable]
        )r!   )r.   rH   ress      r2   r   zProxyLayer.call_loss   s$     >% $.&)C
r3   c                 r    g }| j         D ],}|                    t           |j        |                      -|S )z
        Apply Metrics Function on outputs and labels.

        Args:
            inputs: List[Variable]

        Returns: List[Variable]
        )r"   appendr   compute)r.   rH   outsmetrics       r2   r   zProxyLayer.call_metrics   sE     l 	: 	:FKK 7889999r3   c                 *    || _         |dk    | _        d S )Nr?   )r#   trainingr.   r#   s     r2   set_modezProxyLayer.set_mode   s    	r3   c                 B    t          | j        | j        | j                  S N)r   r    r!   r"   r.   s    r2   clonezProxyLayer.clone   s    $*DNDLIIIr3   c                 &    | j         | j                 S r]   )r%   r#   r^   s    r2   
input_varszProxyLayer.input_vars       	**r3   c                 &    | j         | j                 S r]   )r&   r#   r^   s    r2   
label_varszProxyLayer.label_vars   rb   r3   c                 &    | j         | j                 S r]   )r'   r#   r^   s    r2   rC   zProxyLayer.output_vars        ++r3   c                 &    | j         | j                 S r]   )r(   r#   r^   s    r2   	loss_varszProxyLayer.loss_vars   s    ty))r3   c                 &    | j         | j                 S r]   )r)   r#   r^   s    r2   
loss_nameszProxyLayer.loss_names   rb   r3   c                 &    | j         | j                 S r]   )r*   r#   r^   s    r2   metric_varszProxyLayer.metric_vars   rf   r3   c                 4    | j                                         S r]   )r    _startup_programr^   s    r2   startup_programzProxyLayer.startup_program   s    00222r3   )r-   
__module____qualname____doc__r   r6   jitr   r=   r   r   r   rB   r   r   r[   r_   propertyra   rd   rC   rh   rj   rl   ro   __classcell__)r1   s   @r2   r   r   )   s            < Z& & &
@ @ @.@ @ @4
< 
< 
< 2 2 ]2    ( ( (J J J + + X+ + + X+ , , X, * * X* + + X+ , , X, 3 3 X3 3 3 3 3r3   r   c                   (    e Zd Zd ZddZd Zd ZdS )	BuildInfoc                 .    |                                   d S r]   )clearr^   s    r2   r   zBuildInfo.__init__   s    

r3   Fc                 N    | j         |         }|r|                     |           |S r]   )statescache)r.   r#   updateis_caches       r2   	has_cachezBuildInfo.has_cache   s-    ;t$ 	JJtr3   c                     d| j         |<   d S )NT)r{   rZ   s     r2   r|   zBuildInfo.cache   s     Dr3   c                 8    t          t                    | _        d S r]   )r   boolr{   r^   s    r2   ry   zBuildInfo.clear   s    !$''r3   N)F)r-   rp   rq   r   r   r|   ry    r3   r2   rw   rw      sU             ! ! !( ( ( ( (r3   rw   c                       e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zd ZdS )ProgramHelperz`
    A Helper class for Engine to provides different Program IR according specified 'mode'.
    c                     t          |||          | _        || _        || _        t	                      | _        t          t          j                  | _	        d| _
        i | _        d S )NF)r   proxy_layerinputs_speclabels_specrw   
build_infor   loggingINFO_logger	lazy_init_all_params_dist_attr)r.   r/   r!   r"   r   r   s         r2   r   zProgramHelper.__init__   sY     &eY@@&&#++!',//%'"""r3   c                 t    | j                                          | j                                        | _        dS )z4
        Reset all state of current Object.
        N)r   ry   r   r_   r^   s    r2   resetzProgramHelper.reset   s5     	+1133r3   c                    |dv sJ | j                             |           | j                            |d          r | j                            d| d           dS | j                            d| d           | j        | j        g}t          | 	                                |d          }d	|z   }t          | j         ||           t          | j         |          j        }t          j        j                            d
          d
         s1t#          |j        t'          j                    j        t,                     |                                  dS )z?
        Convert dygraph model into static Program IR.
        r?   rL   rN   Tz"Already build program with mode = z, use cached program.Nz"start to build program for mode = .)
input_spec
full_graph_r@   )r   r[   r   r   r   infor   r   r   static_funcsetattrgetattrconcrete_programr6   rD   rE   rF   r   main_programr   get_instance_amp_recordsr   _build_startup_program)r.   r#   r   r   	func_namer   s         r2   build_programzProgramHelper.build_program  ss    33333!!$'''?$$T400 	LPTPPP   FFtFFFGGG&(89
:$
 
 
 $J	 )[999 #4#3Y??P
 {$../EFF"
 	 # -!.00=#  
 	##%%%%%r3   c                    | j         }t          |                                j                  dk    r	d| _        dS | j        j        D ]C}t          |j        ||j	        |j
        |j        |j        |                                           DdS )zB
        Create and Sync parameters into startup program.
        r   TN)r<   desctypeshapedtypestop_gradientblock)ro   lenglobal_blockopsr   r   
parametersr   r<   r   r   r   r   )r.   ro   params      r2   r   z$ProgramHelper._build_startup_program,  s     .++--122Q66!DNF*5 		 		EZZkk#1%2244    		 		r3   c                 X   |                      |           | j                            dt          |          j                   |j        }d|_        t          | j        | j                  5  |	                    | j
        d                   }ddd           n# 1 swxY w Y   ||_        |S )zD
        Append backward and generate optimizer operations.
        zstart to apply optimizer: %s Nr   )_verify_optimizerr   r   r   r-   _parameter_listr   r   ro   minimizerh   )r.   	optimizeroriginal_paramsrR   s       r2   apply_optimizerzProgramHelper.apply_optimizer@  s     	y)))+T)__-E	
 	
 	
 $3$(	!4,d.BCC 	8 	8$$T^A%677C	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 %4	!
s   +!BBBc                     |J t          |d          s
J d            | j        j        dk    sJ d| j        j         d            t          | j                  dk    sJ dt          | j                               d S )Nr   z&Optimizer must have minimize() method.r?   z(Required mode == 'train', but received ''r   z<Required len(loss_vars) == 1, but received len(loss_vars) = )hasattrr   r#   r   rh   )r.   r   s     r2   r   zProgramHelper._verify_optimizerR  s    $$$y*-- 	
 	
4	
 	
- $///Ot7G7LOOO 0// 4>""a'''`3t~K^K^`` (''''r3   c                     |dv sJ t          | j        d|z             }t          |t                    s
J d            | j                            |           dS )zC
        Switch underly proxy layer mode into target mode.
        r   r   (Please call build_program(mode) firstly.N)r   r   
isinstancer   r[   )r.   r#   funcs      r2   tozProgramHelper.to^  sr     33333t't44$// 	
 	
6	
 	
/ 	!!$'''''r3   c                 z    | j         j        dv s
J d            d| j         j        z   }t          | j         |          S )zJ
        Return StaticFunction instance with underly target mode.
        r   r   r   )r   r#   r   )r.   r   s     r2   r   zProgramHelper.static_funci  sZ     $ )
 
 
 
 6	
 
 

 $*//	t'333r3   c                 	   |                                                                 }i }i }|D ]}|||j        <   | j        j        d         }| j        j        d         }t          t          |                    D ],}	||	         j        |v r||	         j        |||	         j        <   -d}
|D ]T}|                                r| j        |j                 \  }}t                      }||_
        ||_        d}
t          j                    5  t          j        j                            ||          }d d d            n# 1 swxY w Y   |                                r:|                                                    |                                           n(|                                                                 d }||j        |vr(|                                                                 E|                                s[|                                r)||j                 }||         }|                                sJ d|j         d            |                                j        |                                j
        j        |                                j
        j        d}t5                                          |                                          }t9          j        |                                |          }|                    ||           |                                                                 |                                r||j                 }||         }t5                                          |                                          }|                    |                                                                           |                                                                 VtA                      }|
r|j!        dk    rt          j"        #                                dk    rt          j$                     t          j%        dgdd	          }t          j&        j        '                                s"t          j(        )                    ||d
d           t          j*                     d S d S d S d S )Nr   r   FTzparam [z] is not dist tensor typedims_mappingprocess_shapeprocess_groupint32r   ring_id)+r   all_parametersr<   r   r   ranger   is_distr   r   process_meshr   r6   no_gradrD   r   reshard_is_initialized
get_tensor_share_data_with_clearis_denseis_dist_dense_tensor_type	dist_attrr   process_idsr   varr   slice_with_dist_attrnumpysetr   nranksdistributedget_world_sizedisable_staticfullrE   is_compiled_with_xpu_legacy_C_opsbarrierenable_static)r.   r   placeparam_valuesvalue_name_to_valuedy_param_name_to_pir_param_namevalue	dy_params	pir_paramiis_commr   r   r   var_dist_attrtmp
value_namer   pir_scope_paramsliced_paramworld_groupbarrier_tensors                         r2   init_pirzProgramHelper.init_piru  s   #0022AACC *,'! 	4 	4E.3
++)4Q7	)4Q7	s9~~&& 	 	A| $777ENF 0	!0AB  8	, 8	,E}} !-1-GJ.*l !/ 0 0-9*-9*^%% I I +*225-HHCI I I I I I I I I I I I I I I&&(( !$$&&778H8HIIII $$&&--/// E }z!@@@  ""))+++((** ~~ ,<UZH
+J76688  CejCCC 8 %*OO$5$5$B%*__%6%6%C%I%*__%6%6%C%O 	 #/.."4"4Z"@"@"K"K"M"M(=KKMM9     ##L%888  ""))++++ ,<UZH
+J7"..."4"4Z"@"@"K"K"M"M00$$&&1133     ""))+++-//	#"Q&&"1133a77!####[!aw???N#(==?? $,,"NIq    """""	# 	#&&77s   &D::D>	D>	c           	      p   | j         rd S |j        j        }t          j        |                                          }|j        o	|d         dv }d}| j        j        D ]5}|	                                r| j        j
        }	|	                                j        |j                 }
|                    |
          }d}t          j                    5  t          j        j                            ||          }d d d            n# 1 swxY w Y   |                                r:|                                                    |                                           n(|                                                                 d }t          j                                         |G|j        |                                j        vr(|                                                                 |                                s|                                r|                                j        |j                 }
|                    |
          }|j        |j        j        |j        j        d}t?                                           |j                                                  }tC          j"        |#                                |          }|$                    ||           |s&|                                                                 |	                                rqt?                                           |j                                                  }|                    |                                                                           7|r| j        j        D ]}|d         }t?                      %                    |j                  }|s4|j&        t          j'        t          j(        fv rTt?                                           |j                                                  }|r|                                sJ d|j         d            |}|                                r|                                j        |j                 }
|                    |
          }|j        |j        j        |j        j        d}tC          j"        |#                                |          }t          j        j)        *                                5  t          j+        ||j,        	          }d d d            n# 1 swxY w Y   |                                                                 t          j        j)        *                                5  |d
k    rt          j                    5  t          j        j-        .                    |	          5  |/                    t$          j0        j1        j2                  }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   n|dk    rt          j                    5  t          j        j-        .                    |	          5  |/                    t$          j0        j1        j3                  }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                                                                 |	                                r:|                    |                                                                           n'|                    |                                           d d d            n# 1 swxY w Y   ti                      }|r|j5        dk    rt          j6        7                                dk    rt          j8                     t          j9        dgdd          }t          j-        j        :                                s"t          j;        <                    ||dd           t          j=                     d S d S d S d S )Nlevel)o2o3FTr   r   zParameter: z1 is not put into global_scope or not initialized.)r   float16r   bfloat16r   r   r   r   )>r   strategyampcopydeepcopyto_dictenabler   r   r   r   r   varsr<    get_tensor_dist_attr_for_programr6   r   rD   r   r   r   r   r   r   devicesynchronizer   r   r   r   r   r   r   r   r   r   r   find_varr   r   r   dygraphguard	to_tensorr   rE   _dygraph_place_guardcastVarDescVarTypeFP16BF16r   r   r   r   r   r   r   r   r   r   )r.   r   r   dist_contextamp_strategy
amp_configneed_cast_parameterr   r   serial_main_programr   r   r   r   param_tensorr   dense_tensor	amp_dtype	scope_varscope_tensor
param_usedt_castedr   r   s                           r2   initzProgramHelper.init  s	   > 	F#,0]<#7#7#9#9::
*1 
j6I N
 7
 *5 2	O 2	OE}} !&*&;&H#)6688=ejI , M M! ! ^%% I I +*225-HHCI I I I I I I I I I I I I I I&&(( !$$&&778H8HIIII $$&&--/// EM%%''' }z!:!:!<!<!AAA  ""))+++((** ~~ O"//116uzB , M M! ! %2$>%2%?%E%2%?%K 	  ,~~11%*==HHJJ(=KKMM9      u555* 0$$&&--/// O+~~11%*==HHJJ--e.>.>.@.@.K.K.M.MNNN  C	M.9 BM BM&w/	(NN33EJ??	  ;6>6?"CCC+~~11%*==HHJJ  \%A%A%C%C  _%*___ C #
 >>## 0&3355:5:FC$EEcJJ " )6(B)6)C)I)6)C)O! !I $-#Ay$ $L  ,2244  %+%5(& & &
               $$&&--///[(..00 M M I--"N,, "K1FF&+ G    (2&*l&:&? (7 ( (H                              #j00"N,, "K1FF&+ G    (2&*l&:&? (7 ( (H                              ))++22444'')) M$55$//11<<>>    %55h6I6I6K6KLLL=M M M M M M M M M M M M M M M@ .//	#"Q&&"1133a77!####[!aw???N#(==?? $,,"NIq    """""	# 	#&&77s   &C??D	D	T66T:	=T:	
]#$&X
+X	5XXXX	X]#X]#X ]#=&Z1#+Z	Z1ZZ1!Z"Z1%]#1Z55]#8Z59B]##]'	*]'	c                 z    |D ]7}|                                 }|r|j        }|j        }||g| j        |j        <   8d S r]   )r   r   r   r   r<   )r.   
all_paramsparam_valuer   r   r   s         r2   cache_whole_graph_dist_attrz)ProgramHelper.cache_whole_graph_dist_attrd  s^    % 	 	K#--//I (5(5  @*;+;<	 	r3   c                 4    |                                  j        S r]   )r   r   r^   s    r2   r   zProgramHelper.concrete_programo  s    !!22r3   c                     | j         j        S r]   )r   r   r^   s    r2   r   zProgramHelper.main_programs  s    $11r3   c                     	 | j         j        S # t          $ rG}| j                            d           t          |t                    r| j        j        cY d }~S |d }~ww xY w)Nz0The startup_program is not built by `lazy init`.)r   ro   	Exceptionr   warningr   AssertionErrorr   )r.   errs     r2   ro   zProgramHelper.startup_programw  s    	#33 	 	 	L  B   #~.. =,<<<<<<<I	s     
A:AAAAc                 4    t          | j        j                  S r]   )r   r   ra   r^   s    r2   ra   zProgramHelper.input_vars      t'2333r3   c                 4    t          | j        j                  S r]   )r   r   rC   r^   s    r2   rC   zProgramHelper.output_vars      t'3444r3   c                 4    t          | j        j                  S r]   )r   r   rd   r^   s    r2   rd   zProgramHelper.label_vars  r&  r3   c                 4    t          | j        j                  S r]   )r   r   rh   r^   s    r2   rh   zProgramHelper.loss_vars  s    t'1222r3   c                 4    t          | j        j                  S r]   )r   r   rj   r^   s    r2   rj   zProgramHelper.loss_names  r&  r3   c                 4    t          | j        j                  S r]   )r   r   rl   r^   s    r2   rl   zProgramHelper.metric_vars  r(  r3   c                     |                                  }|                    | j        | j                  d         }d |j        D             S )Nc                     i | ]
}|j         |S r   )r<   ).0r   s     r2   
<dictcomp>z2ProgramHelper.named_parameters.<locals>.<dictcomp>  s    GGGe
EGGGr3   )r   get_concrete_programr   r   _params)r.   r   partial_programs      r2   named_parameterszProgramHelper.named_parameters  sU    &&((%::d.
 

 HG/FGGGGr3   N)r-   rp   rq   rr   r   r   r   r   r   r   r   r   r   r  r  rt   r   r   ro   ra   rC   rd   rh   rj   rl   r5  r   r3   r2   r   r      s        ( ( (4 4 4%& %& %&N  (  $

 

 

	( 	( 	(
4 
4 
4Y# Y# Y#vR# R# R#h	 	 	 3 3 X3 2 2 X2 	 	 X	 4 4 X4 5 5 X5 4 4 X4 3 3 X3 4 4 X4 5 5 X5H H H H Hr3   r   )$r   r+   r   collectionsr   r6   r   
paddle.jitr   r   'paddle.jit.dy2static.program_translatorr   r   paddle.jit.dy2static.utilsr	   	paddle.nnr
   paddle.staticr   r   r   paddle.static.amp.fp16_utilsr   r   	converterr   dist_attributer   r   r   r7   r   r   r   rw   r   r   r3   r2   <module>r?     s      # # # # # #        / / / / / / / /        : 9 9 9 9 9       @ @ @ @ @ @ @ @ @ @       
 !           * * * * * * 2 2 2 2 2 2 & & & & & & & &o3 o3 o3 o3 o3 o3 o3 o3d( ( ( ( ( ( ( ("uH uH uH uH uH uH uH uH uH uHr3   