
    x-j(                         d Z ddlZddlZddlZddlZddlmZ  eeej	        d          Z
ddlmZmZmZmZmZmZ ddlmZ dd	lmZmZmZmZmZmZ g Z G d
 d          Z G d d          ZdS )zDefinition of TrainerFactory.    N)
get_loggerz&%(asctime)s-%(levelname)s: %(message)s)fmt   )DownpourLiteDownpourSGDDownpourSGDOPTHeterSectionHogwildSection)Variable)DistMultiTrainerHeterPipelineTrainerHeterXpuTrainerMultiTrainerPipelineTrainerPSGPUTrainerc                        e Zd ZdZd ZddZdS )TrainerFactoryz
    Create trainer and device worker.
    If opt_info is not None, it will get configs from opt_info,
    otherwise create MultiTrainer and Hogwild.
    c                     d S N selfs    [/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/base/trainer_factory.py__init__zTrainerFactory.__init__8   s        Nc                    d }d }|s3t                      }t                      }|                    |           n|                    dd          }|                    dd          } t	                      |                     } t	                      |                     }|X|                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d	          |	                    |d	                    |                    d
          At          |                    d
                    dk    r|                    |d
                    |                    d          At          |                    d                    dk    r|                    |d                    |                    d          |                    |d                    |                    d          At          |                    d                    dk    r|                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          At          |                    d                    dk    r|                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    d|v r|                    |d                    |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d          	 |                    |d                    |                    d          |                    |d                    |                    d          |                    |d                    |                    d           |                     |d                     |                    d!          |!                    |d!                    |                    d"          |"                    |d"                    |                    |           |S )#Ntrainerr   device_workerr
   trainers
trainer_id	dump_slotmpi_rankmpi_sizedump_fieldsr   dump_fields_pathdump_fields_modeuser_define_dump_filenamedump_file_numdump_converter
dump_paramworker_places
use_ps_gpuuse_gpu_graphis_dump_in_simple_modedump_num_decimalsenable_random_dumpdump_intervalrandom_with_lineid
fleet_descuse_cvmno_cvm%scale_sparse_gradient_with_batch_sizescale_datanormadjust_ins_weight
copy_tablecheck_nan_var_names
loss_names)#r   r
   _set_device_workergetglobals_set_trainers_set_trainer_id_set_dump_slot_set_mpi_rank_set_mpi_sizelen_set_dump_fields_set_dump_fields_path_set_dump_fields_mode_set_user_define_dump_filename_set_dump_file_num_set_dump_converter_set_dump_param_set_worker_places_set_use_ps_gpu_set_use_gpu_graph_set_is_dump_in_simple_mode_set_dump_num_decimals_set_enable_random_dump_set_dump_interval_set_random_with_lineid_set_fleet_desc_set_use_cvm_set_no_cvm&_set_scale_sparse_grad_with_batch_size_set_scale_datanorm_set_adjust_ins_weight_set_copy_table_config_set_check_nan_var_names_set_loss_names)r   opt_infor   r   trainer_classdevice_worker_classs         r   _create_trainerzTrainerFactory._create_trainer;   s    h	6"nnG#IIM&&}5555$LLNCCM"*,,	"J"J.gii.00G:GII&9:<<M #<<
++7))(:*>???<<--9++H\,BCCC<<,,8**8K+@AAA<<
++7))(:*>???<<
++7))(:*>???LL//;HLL7788A==,,Xm-DEEELL!344@HLL);<<==BB11(;M2NOOO<< 233?11(;M2NOOOLL!<==IHLL)DEEFF!KK:: !<=   <<00<..x/HIII<< 011=//9I0JKKKLL..:HLL66771<<++H\,BCCC<<00<..x/HIII<<--9++H\,BCCC<<00<..x/HIII<< 899E77 !9:   << 344@22 !45   << 455A33 !56   <<00<..x/HIII<< 455A33 !56   x''--h|.DEEE''(>???<<	**6(())<===<<))5''(:;;;LL!HII  BB !HI   << 011=//9I0JKKK<< 344@22 !45   <<--9228L3IJJJ<< 566B44 !67   <<--9++H\,BCCC&&}555r   r   )__name__
__module____qualname____doc__r   ra   r   r   r   r   r   1   sG           l l l l l lr   r   c                   *    e Zd ZdZd Zd Zd Zd ZdS )FetchHandlerMonitorzN
    Definition of FetchHandlerMonitor class,
    it's for fetch handler.
    c                     || _         t          j        | j        || j         f          | _        t          j                    | _        d| _        d S )N)targetargsF)fetch_instance	threadingThreadhandler_launch_funcfetch_threadLockrunning_lockrunning)r   scopehandlers      r   r   zFetchHandlerMonitor.__init__   sQ    %%,+5$:M2N
 
 
 &N,,r   c                    |}|j         }i }|j        D ][}t          |j        |         t                    r|||j        |         j        <   8t
                              d| d           ||d<   \d}	 | j                                         | j	        du rd S ||k     rt          j        d           |dz  }nd}i }|D ]A}|                    |          }	|	||<   |	#t
                              ||          d           Bi }
|D ]}||         }||         d |
|<   ||                                         |
|<   |
|                                         }t          |          dk    rt!          d	          |
|                                         rt%          j        |
|                   |
|<   d |
|<   |                    |
           | j                                         p)
Nzthe value of z is not a VariablezNone.varr   TFr   z value currently not availablea  Some of your fetched tensors                                             hold LoD information.                                             They can not be completely cast                                             to Python ndarray. We can                                             not return DenseTensor itself directly,                                             please choose another targets)period_secsvar_dict
isinstancer   namelocal_loggerwarningrq   acquirerr   timesleepfind_var
get_tensorlodrE   RuntimeError_is_initializednparrayrt   release)r   rs   rt   rk   rv   var_name_to_keykeyelapsed_secs
fetch_dictvarres_dict	user_namer   s                r   rn   z'FetchHandlerMonitor.handler_launch_func   s=    $0!* 	2 	2C.1#6AA 2EH 7 < ABB$$%LS%L%L%LMMM.1
++*	(%%'''|u$$k))
1! 
*  C..--C&)JsO{$,,.s3SSS   % 3 3C / 4I!#..2+ .8o.H.H.J.J+"9-1133C3xx!||*K    	*::<< 3.0hx	7J.K.K++.2++&&x000%%'''U*	(r   c                     | j                                          d| _        | j                                          | j                            d           | j                                         dS )zH
        start monitor,
        it will start a monitor thread.
        TN)rq   r|   rr   r   ro   	setDaemonstartr   s    r   r   zFetchHandlerMonitor.start   sg    
 	!!###!!#####D)))!!!!!r   c                 x    | j                                          d| _        | j                                          d S )NF)rq   r|   rr   r   r   s    r   stopzFetchHandlerMonitor.stop   s9    !!###!!#####r   N)rb   rc   rd   re   r   rn   r   r   r   r   r   rg   rg      s[         
  5( 5( 5(n	" 	" 	"$ $ $ $ $r   rg   )re   loggingrl   r}   numpyr   paddle.base.log_helperr   rb   INFOrz   r   r   r   r   r	   r
   r   	frameworkr   trainer_descr   r   r   r   r   r   __all__r   rg   r   r   r   <module>r      s   $ #           - - - - - -zgl H                                        v v v v v v v vrS$ S$ S$ S$ S$ S$ S$ S$ S$ S$r   