
    x-jT                        d dl Z ddlmZ  e             d dlmZmZ d dlmZmZ d dl	m
Z
mZ ddlmZmZmZ ddlmZ dd	lmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# dd
l$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 ddl:m;Z; ddl<m=Z= ddl>m?Z?m@Z@mAZA ddlBmCZC ddlDmEZEmFZFmGZGmHZHmIZI ddlJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZf ddlgmhZhmiZimjZj ddlkmlZl ddlmmnZnmoZo ddlpmqZq ddlrmsZsmtZtmuZumvZv ddlwmxZx ddlymzZzm{Z{m|Z|m}Z}m~Z~ ddlmZmZmZ ddlmZmZ ddlmZ g d ZdS )!    N   )monkey_patch_value_in_dist)	Placement
ReduceType)ParallelModecreate_nccl_config)InMemoryDatasetQueueDataset)cloud_utilsiorpc)shard_op)DistAttr	DistModelShardingStage1ShardingStage2ShardingStage3Strategydtensor_from_fnenable_auto_dpin_auto_parallel_align_modereshardshard_dataloadershard_layershard_optimizershard_scalershard_tensor	to_staticunshard_dtensor)to_distributed)get_meshset_mesh)ContextParallelPrepareContextParallel)parallelize)
SplitPoint)	ColWiseParallelConvParallelPrepareLayerInputPrepareLayerOutputRowWiseParallelSequenceParallelBeginSequenceParallelDisableSequenceParallelEnableSequenceParallelEnd)
LocalLayer)	local_map)Partial	ReplicateShard)ProcessMesh)is_available	new_grouprestart_process_groupshutdown_process_groupsplit)P2POpReduceOp
all_gatherall_gather_object
all_reducealltoallalltoall_singlebarrierbatch_isend_irecv	broadcastbroadcast_object_listdestroy_process_groupgatherget_backend	get_groupirecvis_initializedisendrecvrecv_object_listreducereduce_scatterscatterscatter_object_listsendsend_object_liststreamwait)CountFilterEntryProbabilityEntryShowClickEntry)BoxPSDataset)load_merged_state_dictload_state_dict)save_state_dict)ShardedStateDictShardedWeightbuild_sharded_state_dictshard_weight)launch)DataParallelParallelEnvget_rankget_world_sizeinit_parallel_env)gloo_barriergloo_init_parallel_envgloo_release)group_sharded_parallelsave_group_sharded_model)spawn)]r   rm   rb   rQ   rG   rR   rD   rE   rd   r7   r9   r8   rg   ri   rh   rj   r
   r:   rW   rY   rf   rI   r=   r>   r	   rB   r?   r@   rA   rS   rO   rM   r<   rV   re   rX   r   rK   rF   rL   rJ   rT   rN   rP   r6   rH   r5   r   r   r   r   r   r   r   r   r4   r3   r2   r]   r\   r[   r   r   r   r   r   r   r   r   r0   r1   r   r%   r/   r,   r.   r-   r'   r+   r*   r)   r&   r"   r!   r    r(   r#   r$   r   r_   r^   ra   r`   )atexitvalue_patchr   paddle.base.corer   r   &paddle.distributed.fleet.base.topologyr   r    paddle.distributed.fleet.datasetr	   r
    r   r   r   auto_parallelr   auto_parallel.apir   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   auto_parallel.high_level_apir    auto_parallel.interfacer!   r"   +auto_parallel.intermediate.context_parallelr#   r$   &auto_parallel.intermediate.parallelizer%   ,auto_parallel.intermediate.pipeline_parallelr&   *auto_parallel.intermediate.tensor_parallelr'   r(   r)   r*   r+   r,   r-   r.   r/   auto_parallel.local_layerr0   auto_parallel.local_mapr1   auto_parallel.placement_typer2   r3   r4   auto_parallel.process_meshr5   
collectiver6   r7   r8   r9   r:   communicationr;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   
entry_attrrW   rX   rY   fleetrZ   #flex_checkpoint.dcp.load_state_dictr[   r\   #flex_checkpoint.dcp.save_state_dictr]   "flex_checkpoint.dcp.sharded_weightr^   r_   r`   ra   launch.mainrb   parallelrc   rd   re   rf   rg   parallel_with_gloorh   ri   rj   shardingrk   rl   rm   __all__     [/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/__init__.py<module>r      s    3 3 3 3 3 3      2 2 2 2 2 2 2 2        K J J J J J J J         
 $ # # # # #                                     & 9 8 8 8 8 8 7 7 7 7 7 7 7 7        @ ? ? ? ? ? D D D D D D
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 2 1 1 1 1 1 . . . . . .         
 4 3 3 3 3 3                                                                        <         
               A @ @ @ @ @                                        
             ^ ^ ^r   