
    IЦi2              	          % S SK r S SKJrJrJrJrJr  S SKrS SKJ	r
  S SKJr  S SKJr  S SKJr  S SKJr  S SKJr  SS	KJrJrJrJrJrJrJr  SS
KJrJrJ r J!r!J"r"J#r#  SSK$J%r%J&r&  S/r'\\(   \)S'   S\RT                  S\4S jr+S\S\4S jr,S\S\S\!4S jr-S\(S\S\"4S jr.S\(S\S\S\"4S jr/S\(S\RT                  S\"4S jr0S\(S\4S jr1S r2S r3S\(S\S\\   S\\   4S jr4S \S\ 4S! jr5S\(S"\S\\"   4S# jr6S\S\4S$ jr7S\RT                  S\\   4S% jr8S\(S&\S'\S\\   4S( jr9S \\(\4   S\4S) jr:S*\S+\S,\S-\4S. jr;g)/    N)AnyCallablecastDictList)_get_device_module)ShardMetadata)ShardedTensor)DTensor)%compute_local_shape_and_global_offset   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorPropertiesTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__tensorreturnc           	          [        [        R                  " S/[        U R	                  5       5      -  5      U R	                  5       S9$ )Nr   offsetssizes)r   torchSizelensize)r   s    k/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensorr*   '   s5    

A3V[[]!334FKKM     shard_mdc                     [        [        R                  " U R                  5      [        R                  " U R                  5      S9$ Nr"   )r   r%   r&   shard_offsetsshard_sizes)r,   s    r)   _chunk_for_shardr1   -   s3    

8112jj--. r+   sharded_tensorc                    U R                  5       R                  n[        UR                  UR                  UR
                  UR                  UR                  S9n[        [        U5      UU R                  5       R                  S9$ )N)dtypelayoutrequires_gradmemory_format
pin_memorychunk
propertiesr(   )metadatatensor_propertiesr   r4   r5   r6   r7   r8   r   r1   r(   )r2   r,   shard_propertiesr;   s       r)   _sharded_tensor_metadatar?   4   s|     &..0BB!$$&&&44&44#..J x($$&++ r+   fqnc                 x   [        UR                  UR                  UR                  5      u  p#[        R
                  " U5      [        R
                  " U5      p2[        [        X5      [        R                  [        [        UUS9[        R                  " UR                  5       5      UR                  5       S9S9$ )Nr"   r9   indextypetensor_data)r   shapedevice_mesh
placementsr%   r&   r   r   r   SHARDr   r   r   create_from_tensorto_localr(   )r@   r   r$   r#   s       r)   _create_write_items_for_dtensorrL   H   s    :f((&*;*;NE ZZ&

7(;7C)  #& (::6??;LM
 r+   c                     [         R                  " UR                  5      n[        [	        X5      [
        R                  [        X5      S9$ )NrB   )r%   r&   r/   r   r   r   rI   r?   )r@   r2   r,   r#   s       r)   _create_write_item_for_shardrN   \   s>     jj//0GC)  ,^F r+   c                 (   [         R                  " S/[        UR                  5       5      -  5      n[	        [        X5      [        R                  [        [        X!R                  5       S9[        R                  " U5      UR                  5       S9S9$ )Nr   r"   r9   rB   )r%   r&   r'   r(   r   r   r   TENSORr   r   r   rJ   )r@   r   r#   s      r)   _create_write_item_for_tensorrQ   g   sm    jj!s6;;=112GC)!!#&wkkmL'::6B
 r+   bytesc                 D    [        [        U 5      [        R                  S9$ )N)rC   rD   )r   r   r   BYTE_IO)r@   rR   s     r)   _create_write_item_for_bytesiorU   t   s     C "" r+   c           
          [        [        R                  U [        R                  " U45      U[        R                  " U45      [        R                  " U45      S9$ N)rD   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rT   r%   r&   rX   dest_offsetrZ   storage_offsetlengths        r)   _create_read_item_for_byteiora   {   sK     !!ZZ/#

N#45

F9% r+   c           
          [        [        R                  U [        R                  " U5      U[        R                  " U5      [        R                  " U5      S9$ rW   )r   r   rP   r%   r&   rX   rY   rZ   r[   r\   s        r)   _create_read_item_for_tensorrd      sD       ZZ-#

?3

7# r+   checkpoint_mdlocal_chunksc                    / n[        U5       H  u  pE[        UR                  5       H  u  pg[        XW5      (       d  M  / n/ n	/ n
[        XuS9 H;  u  nnnnUR	                  U5        U	R	                  U5        U
R	                  U5        M=     UR	                  [        [        XR                  U5      U	[        XR                  U5      UU
S95        M     M     U$ )a'  
Create a list of ``ReadItem`` based on the checkpoint and local chunks.

This applies the resharding algorithm and computes the reads needed
to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

Args:
    fqn (str) : The state_dict FQN to pass to ``ReadItem``.
    checkpoint_md (TensorStorageMetadata): metadata for a given tensor
        from a checkpoint.
    local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
        loaded.

Returns:
    A list of ``ReadItem`` that will satisfy all input chunks.
)saved_shardcurrent_shardrc   )	enumeratechunksr   r   appendrd   r   r#   )r@   re   rf   
read_itemsidxshardstorage_idx
storage_mdr[   rY   r\   _dimoffset_for_saved_tensoroffset_for_current_tensorr`   s                  r)   r   r      s    * J-
'01E1E'F#K5eHH OLG =&')  &&'>?##$=>v& ,,S--E!-"/5G5G"U$3#' (G .: r+   
state_dictc                   ^^ / nU R                  5        H  u  mm[        T[        5      (       a  UR                  [	        TT5      5        M8  [        T[
        5      (       a6  UR                  UU4S jTR                  5       R                   5       5        M  [        T[        R                  5      (       a  UR                  [        TT5      5        M  UR                  [        TT5      5        M     [        U5      $ )Nc              3   >   >#    U  H  n[        TTU5      v   M     g 7f)N)rN   ).0r,   r@   objs     r)   	<genexpr>5_create_default_metadata_only_plan.<locals>.<genexpr>   s$       >H -S#x@@ >s   )items
isinstancer   rl   rL   r
   extendr<   shards_metadatar%   TensorrQ   rU   r   )ru   requestsr@   ry   s     @@r)   "_create_default_metadata_only_planr      s    H$$&Sc7##OO;CEF]++OO  # > >  U\\**OO9#sCDOO:3DE ' Hr+   objectc                 R   [        US5      (       a  UR                  X5      $ [        U[        5      (       a5  UR	                  5        Vs/ s H  n[        XUR                  5      PM     sn$ [        U[        R                  5      (       a  [        X5      /$ [        X5      /$ s  snf )N__create_write_items__)hasattrr   r}   r
   local_shardsrN   r<   r%   r   rQ   rU   )r@   r   ro   s      r)   _create_write_itemsr      s    v/00,,S99	FM	*	*  ,,.
. )ennE.
 	
 
FELL	)	)-c:;;.s;<<
s   
 B$c                     [        U R                  U R                  U R                  5      u  p[        R
                  " U5      [        R
                  " U5      p![        UUS9$ r.   )r   rF   rG   rH   r%   r&   r   )r   r$   r#   s      r)   _create_chunk_from_dtensorr      sS    :f((&*;*;NE ZZ&

7(;7 r+   c                 n   [        U S5      (       a  U R                  5       nU$ [        U [        5      (       a5  U R	                  5        Vs/ s H  n[        UR                  5      PM     nnU$ [        U [        R                  5      (       a  [        U 5      /nU$ [        S[        U 5       35      es  snf )N__create_chunk_list__zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )r   r   r}   r
   r   r1   r<   r%   r   r*   
ValueErrorrD   )r   rf   ro   s      r)   _create_chunk_listr      s    v.//335  
FM	*	*:@:M:M:O
:OU^^,:O 	 
  
FELL	)	)1&9:  V~'
 	

s   B2mdry   c                     [        U[        5      (       d   [        U5      n[        XU5      $ [        [        U 5      S[        U 5      SSS9/$ ! [         a$  n[        SU  S3S[	        U5       3-   5      UeS nAff = f)Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   r]   )r}   r   r   r   rD   r   ra   r   )r@   r   ry   rf   exs        r)   _create_read_itemsr   
  s    b.//	-c2L 0FF )(-+C0 
 	
  	23%r:<T"XJGH 	s   A 
A<A77A<c                 v    S[         4S jnS[        4S jnS[        R                  4S jn[	        U UUU5        g)zH
Initializes meta tensor if the meta tensor is DTensor or torch.Tensor.
valuec                    [        U SS 5      nU[        R                  " S5      :X  a  [        R                  R                  5       R                  n[        [        R                  [        U5      R                  5       5      n[        R                  " U R                  5       US9n[        R                  " UU R                  U R                  U R!                  5       U R#                  5       S9nU$ U $ )Ndevicemetar   )rG   rH   rF   stride)getattrr%   r   distdistributed_c10d_get_pg_default_devicerD   r   r   current_device
empty_likerK   r   
from_localrG   rH   r(   r   )r   r   device_typenew_local_tensordtensors        r)   dtensor_func&_init_state_dict.<locals>.dtensor_func&  s    $/U\\&))//FFHMMK0=LLNF  %//0@P (( !-- ++jjl||~G NLr+   c                     [        U SS 5      nU[        R                  " S5      :X  a  [        S[	        U 5       S35      eU $ )Nr   r   zFound unsupported type z for meta device loading.)r   r%   r   RuntimeErrorrD   )r   r   s     r)   sharded_tensor_func-_init_state_dict.<locals>.sharded_tensor_func;  sF    $/U\\&)))$u+6OP  Lr+   c                 2   [        U SS 5      nU[        R                  " S5      :X  ao  [        R                  R                  5       R                  n[        [        R                  [        U5      R                  5       5      n[        R                  " XS9nU$ U $ )Nr   r   r   )r   r%   r   r   r   r   rD   r   r   r   r   )r   r   r   r   s       r)   tensor_func%_init_state_dict.<locals>.tensor_funcD  sy    $/U\\&))//FFHMMK0=LLNF %%e;FMLr+   N)r   r   r%   r   _iterate_state_dict)ru   r   r   r   s       r)   _init_state_dictr   !  s@    
G *3 
5<< 
 	r+   iter_objectr   r   r   c           	      p   [        U [        5      (       a  U" U 5      $ [        U [        5      (       a  U" U 5      $ [        U [        R                  5      (       a  U" U 5      $ [        U [
        [        [        [        [        R                  45      (       d  U c  U $ [        U [        5      (       a)  U R                  5        H  u  pE[        XQX#5      X'   M     U $ [        U [        [        45      (       a>  U  Vs/ s H  n[        XaX#5      PM     nn[        U [        5      (       a  [        U5      nU$ gs  snf )a  
Iterate through the state dict, applying the given functions to each tensor type
and update the state dict in place.

Args:
    iter_object (Any): the target state_dict.
    sharded_tensor_func (Callable): the function to apply to ShardedTensor
    dtensor_func (Callable): the function to apply to DTensor
    tensor_func (Callable): the function to apply to Tensor

# TODO: let state_dict_util._iterate_state_dict() to support in place option
so we don't need to have two versions of _iterate_state_dict.
N)r}   r   r
   r%   r   intfloatstrrR   ioBytesIOdictr|   r   listtuple)r   r   r   r   keyr   vrets           r)   r   r   X  s   ( +w''K((	K	/	/";//	K	.	.;'';eS% DEE	K	&	&%++-JC2%8 K . 	K$	/	/ !
   1DR  	 
 k5))*C
 
0
s   8D3)<r   typingr   r   r   r   r   r%   torch.distributeddistributedr   torch._utilsr   !torch.distributed._shard.metadatar	   'torch.distributed._shard.sharded_tensorr
   torch.distributed.tensorr   torch.distributed.tensor._utilsr   r<   r   r   r   r   r   r   r   plannerr   r   r   r   r   r   
reshardingr   r   r   r   __annotations__r   r*   r1   r?   rL   rN   rQ   rU   ra   rd   r   r   r   r   r   r   r   r    r+   r)   <module>r      s   	 2 2    + ; A , Q    99c 9ell 7K } 1E !-:( g ) (	+7D
s 
ELL 
Y 
 C 

4	4(4 +,4 
(^	4n? x "=S =# =$y/ =w 3G u|| 5I0J &
C 
] 
 
h 
.4c3h 4C 4n,,, ", 	,r+   