
    Αi                     Z    S SK r S SKJr  SSKJr  SSKJrJrJrJ	r	J
r
JrJrJrJr  SqS rg)    N)fleet   )ParallelMode)	DualPipeVParallelPipelineLayerPipelineParallelPipelineParallelWithInterleave$PipelineParallelWithInterleaveFthenBSegmentParallelShardingParallelTensorParallelVPPFhenBInBalancedMemoryc           	      X   [         R                   nU c   S5       e[        R                  R                  5       S::  a  U $ UR                  nUR
                  (       a  UR                  S   (       d  UR                  S   (       a  SOSnUS:X  a8  [        R
                  R                  U SSSSUR                  S   (       a  SOS	S
9n UR                  S   nUR                  S   nUR                  S   nUR                  S   nUR                  S   nUR                  S   n	[        R
                  R                  UUUUUU	S9q	UR                  (       a7  [        R                  " U UR                  UR                  UR                  S9n
U
$ UR                  R!                  5       ["        R$                  :X  a  ['        XR                  US9n U $ UR                  R!                  5       ["        R(                  :X  aP  [        R                  " U UR                  UR                  UR                  UR                  R+                  5       S9n U $ UR                  R!                  5       ["        R,                  :X  a  [/        XR                  US9n U $ UR                  R!                  5       ["        R0                  :X  a  [3        XR                  US9n U $ UR                  R!                  5       ["        R4                  :X  Ga1  [7        U [8        5      (       d   S5       eUR:                  S   R<                  (       a  [?        XR                  US9n U $ U RA                  5       S:X  a  [C        XR                  US9n U $ URD                  S   nUR                  RG                  5       nUSU-  :  a  [I        XR                  US9n U $ Xs=::  a	  SU-  :  aN  O  OKUR:                  S   RJ                  (       a  [M        XR                  US9n U $ [O        XR                  US9n  U $ [Q        SU SU S35      eU $ )a  
Return distributed data parallel model (Only work in dygraph mode)

Args:
    model (Layer): the user-defined model which inherits Layer.

Returns:
    distributed data parallel model which inherits Layer.

Examples:

    .. code-block:: python

        >>> import paddle
        >>> import paddle.nn as nn
        >>> from paddle.distributed import fleet

        >>> class LinearNet(nn.Layer):
        ...     def __init__(self):
        ...         super().__init__()
        ...         self._linear1 = nn.Linear(10, 10)
        ...         self._linear2 = nn.Linear(10, 1)
        ...     def forward(self, x):
        ...         return self._linear2(self._linear1(x))

        >>> # 1. initialize fleet environment
        >>> fleet.init(is_collective=True)

        >>> # 2. create layer & optimizer
        >>> layer = LinearNet()
        >>> loss_fn = nn.MSELoss()
        >>> adam = paddle.optimizer.Adam(
        ...     learning_rate=0.001, parameters=layer.parameters())

        >>> # 3. get data_parallel model using fleet
        >>> adam = fleet.distributed_optimizer(adam)
        >>> dp_layer = fleet.distributed_model(layer)

        >>> # 4. run layer
        >>> inputs = paddle.randn([10, 10], 'float32')
        >>> outputs = dp_layer(inputs)
        >>> labels = paddle.randn([10, 1], 'float32')
        >>> loss = loss_fn(outputs, labels)
        >>> print("loss:", loss.numpy())
        >>> loss.backward()
        >>> adam.step()
        >>> adam.clear_grad()


Nzmodel should not be Noner   use_pure_fp16use_pure_bf16O2O1float16bfloat16)models
optimizerslevelmaster_weight
save_dtypedtypeinit_loss_scaling
incr_ratio
decr_ratioincr_every_n_stepsdecr_every_n_nan_or_infuse_dynamic_loss_scaling)r   r   r   r   r    r!   )comm_buffer_sizelast_comm_buffer_sizefind_unused_parameters)strategy)r"   r#   r$   groupzDFor pipeline parallel, the model should an instance of PipelineLayer
pp_configsaccumulate_steps   zThe accumulate_steps(z/) should be greater than or equal to pp_degree()))r   paddledistributedget_world_size_user_defined_strategyampamp_configsdecorate
GradScaler_grad_scalarheter_ccl_modeDataParallelfuse_grad_size_in_MBlast_comm_group_size_MBr$   _hcgget_parallel_moder   SHARDING_PARALLELr   DATA_PARALLELget_data_parallel_groupSEGMENT_PARALLELr   TENSOR_PARALLELr   PIPELINE_PARALLEL
isinstancer   hybrid_configsuse_dualpipevr   get_num_virtual_stagesr   pipeline_configsget_pipe_parallel_world_sizer	   best_unbalanced_schedulerr   r
   
ValueError)model	fleet_envr%   r   r   r   r   r   r    r!   distributed_modelr(   	pp_degrees                ^/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/fleet/model.pyrJ   rJ   "   s   f I888((*a///H|| ##O4##O4  	 	 D=JJ''"  ++O< # ( E %001DE)),7
)),7
%112FG"*"6"6%#
 $,#7#7&$
 
 zz,,/!!1$;%= - 
 "//%::"*"B"B#+#B#B	
 ! ~~'')\-K-KK J\ L[ 
	)	)	+|/I/I	I##%::"*"B"B#+#B#B..88:
X LK 
	)	)	+|/L/L	L~~IH LG 
	)	)	+|/K/K	KunnxHD LC 
	)	)	+|/M/M	M%// 	
R	
/ ""<0>>%e^^hOE8 L7 ))+q0$UNNXNE2 L/  (889KL!CCEI1y=06>>H& L! >Y>** ++, 5~~E L A~~E L	 !+,<+==lmvlwwxy  L    )r+   paddle.distributedr   base.topologyr   meta_parallelr   r   r   r	   r
   r   r   r   r   r3   rJ    rM   rL   <module>rR      s,     $ '
 
 
 ]rM   