
    x-j^                    l   d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlZd dlmZ dd	lmZ d
dlmZmZmZ d
dlmZmZ erd dlmZ d9dZd Zd:dZ d Z!d Z"d Z#d  Z$d! Z%d" Z&d# Z'd$ Z(d;d&Z)d' Z*	 d<d(Z+d) Z,d* Z-d+ Z.d, Z/d- Z0d. Z1d=d1Z2	 d<d>d4Z3d5 Z4d6 Z5d7 Z6d8 Z7dS )?    )annotationsN)defaultdictreplace)TYPE_CHECKING)	safe_open)logger   )postprocess_transpose   )LocalTensorIndexLocalTensorMetadataMetadata)ShardedWeightShardedWeightDesc)coremeshnp.array | list[list[int]]rankintc                    t          j        |           } | |k                                    }|j        d         dv sJ d|j                     |j        d         dk    r|d                                         nd S )Nr   )r   r   zrand_coordinator.shape: )paddle	to_tensornonzeroshapetolist)r   r   rand_coordinators      l/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/flex_checkpoint/dcp/utils.pyget_coordinatorr   /   s    D!!D--//!!$ )    ;"2"8::  
 )9(>q(AA(E(E""$$$4    c                f    | |z  dk    }| |z   dz
  |z  g|z  }|r|d         }|||z  | z
  z
  |d<   |S )Nr   r    )
total_numsnum_of_pieceshas_remainderresult
last_values        r   balanced_splitr)   <   s\    .!3MM)A--?@=PF LBZ
:#=
#JKr
Mr    global_shape	list[int]process_meshcore.ProcessMesh
placementslist[core.Placement]returntuple[tuple[int], tuple[int]]c                F   ddl m} t          j        |j                                      |j                  }t          j        	                                |vrdS t          |t          j        	                                          }t          j        |           }d | D             } ||t          |                     \  }}	t          |          D ]\  }
}t          |          dk    rdgt          |           z  }|D ]}||         }t          ||
         |j        |                   }||         ||
<   t          |d |                   ||
<   ||
         ||
         k    r||
         ||
<   n||
xx         ||
         z  cc<   t!          |          t!          |          fS )Nr   )placemetns_to_dist_statusNNc                    g | ]}d S r   r#   ).0_s     r   
<listcomp>z9compute_local_shape_and_global_offset.<locals>.<listcomp>T   s    ---1Q---r    )/paddle.distributed.auto_parallel.placement_typer3   nparrayprocess_idsreshaper   r   distributedget_rankr   copylen	enumerater)   sumtuple)r*   r,   r.   r3   r   rank_coordinatorlocal_shapeglobal_offsetdims_mappingr8   
tensor_dim	mesh_dimslocal_offsetmesh_dim	chunk_idxchunkss                   r   %compute_local_shape_and_global_offsetrP   E   s   
      8L,--55l6HIID""$$D00|&tV-?-H-H-J-JKK)L))K-----M//
C<M<MNNOL!!*<!8!8 F F
Iy>>QsS...! 	F 	FH(2I#J');H)E F '-Y&7K
#'*6*9*+='>'>L$Z(L,DDD,8,Dj))j)))\*-EE))))	F u]3333r    c                >    i i fd d|            fS )z_
    Flatten the nested dict to a flat dict.
    {"model": {"w0": xxx}} -> {model.w0: xxx}
    c                t   t          |t                    rO|                                D ]8\  }}t          |t                    sJ d|              g | |R |           9d S t          |t          j        t          f          r!d                    |           }||<   | |<   d S t          d|           )NzThe key should be str, but is .z2The value should be dict or paddle.Tensor, but is )	
isinstancedictitemsstrr   Tensorr   join
ValueError)keyvaluekvflatten_key_str_flattenflatten_state_dictmappings        r   r`   z$flatten_state_dict.<locals>._flattens   s    eT"" 	 ' '1!!S))OO+OA+O+OOO)3A&&&&' ' }=>> 	!hhsmmO27/'*GO$$$LULL  r    r#   r#   )
state_dictr`   ra   rb   s    @@@r   ra   ra   k   sT    
 G       HRw&&r    c                ,   i }|                                  D ]|\  }}||         }t          |t                    sJ d|             |}t          t	          |          dz
            D ] }||         }|                    |i           }!|||d         <   }|S )za
    Unflatten the flat dict to a nested dict.
    {model.w0: xxx} -> {"model": {"w0": xxx}}
    z The key should be tuple, but is r   r"   )rV   rT   rE   rangerB   
setdefault)flat_state_dictrb   rc   r[   r\   	key_tupletmpis           r   unflatten_state_dictrk      s    
 J%++-- 	# 	#
UCL	)U++ 	
 	
:y::	
 	
+ s9~~)** 	* 	*AA,C..b))CC"IbMr    c                    dg}t          j        d          }t          j        |           }|D ]N}|                    |          }|r5|                    t          |                    d                               O|rt          |          nd S )Nr   z^(\d+)_(\d+)\.distcp$r
   )	recompileoslistdirmatchappendr   groupmax)pathnumberspatternfilesfilerq   s         r   
get_max_idrz      s    cGj122GJtE 0 0d## 	0NN3u{{1~~..///",3w<<<,r    c                    g }t           j                            || |           |dd          D ]}||d         k    sJ d| d            d S )Nr   r   zid:z !=  all_unique_id[0])r   r?   all_gather_object)	unique_idprocess_groupall_unique_idids       r   check_unique_idr      s~    M
((y-   ABB G G]1%%%%'FR'F'F'F%%%%G Gr    c                F    d}t          | |          D ]\  }}||z  |z   }|S )Nr   )zip)indicesr   idxrj   dims        r   ravel_indexr      s8    
Cgu%%  3Ci!mJr    c                    g }t          |          D ]}|                    | |z             | |z  }  t          t          |                    S N)reversedrr   rE   )r   r   r   r   s       r   unravel_indexr      sS    G  sSy!!!'""###r    c                   t          ||           }t          |dz
  |           }g }t          t          |                     D ]}|dk    r||         }||         dz   }n^||dz
           ||dz
           k    r<t          ||         ||                   }t	          ||         ||                   dz   }n
d}| |         }|                    ||f           |||fS )Nr   r   )r   re   rB   minrt   rr   )	r   
flat_startflat_end	start_idxend_idx
min_slicesaxisses	            r   minimal_nd_slicer      s    j%00IHqL%00GJc%jj!! " "199$A!AA"gdQh&777	$77	$77!;$K1a&!!!!y'))r    c                h    t          d |D                       }t          ||           }||z
  ||z
  fS )Nc              3  &   K   | ]}|d          V  dS r   Nr#   )r7   r   s     r   	<genexpr>z*flat_range_in_min_slice.<locals>.<genexpr>   s&      00qt000000r    )rE   r   )r   r   r   r   
min_startsmin_flat_starts         r   flat_range_in_min_slicer      sB    00Z00000J U33N&>(AAAr    Tc                    t          |                                           }t          d |D                       }t          d |D                       }|r|st	          d          |s|S |rd}nt          |          dk    rd}nd}g }t          j                            |||           t          d |D                       s#t          d |D                       s
J d	            t          d
 |D                       S )Nc              3  @   K   | ]}t          |t                    V  d S r   rT   r   r7   r^   s     r   r   z(is_sharded_state_dict.<locals>.<genexpr>   s,      FF!A}55FFFFFFr    c              3  @   K   | ]}t          |t                    V  d S r   r   r   s     r   r   z(is_sharded_state_dict.<locals>.<genexpr>   s,      CCqjM22CCCCCCr    z?All values must be ShardedWeight if any value is ShardedWeight.r   r   r"   c              3  "   K   | ]
}|d k    V  dS r   r#   r7   fs     r   r   z(is_sharded_state_dict.<locals>.<genexpr>   &      ))!qAv))))))r    c              3  "   K   | ]
}|d k    V  dS r   r#   r   s     r   r   z(is_sharded_state_dict.<locals>.<genexpr>   s&      0K0KAa0K0K0K0K0K0Kr    zUNot support mixed type of ShardedWeight and non-ShardedWeight in the same state_dict!c              3  "   K   | ]
}|d k    V  dS r   r#   r   s     r   r   z(is_sharded_state_dict.<locals>.<genexpr>   r   r    )	listvaluesallany	TypeErrorrB   r   r?   r|   )rc   use_distr~   r   is_all_shardedhas_shardedflag	all_flagss           r   is_sharded_state_dictr      sD   *##%%&&FFFvFFFFFNCCFCCCCCK 
> 
M
 
 	
   	V		I
((D-HHH))y))))) S0K0K0K0K0K-K-K  _ K ))y))))))r    c                   t          |           }g }g }g }g }t          |          D ]}	| |	         }
| |	         ||	         z   }||	         }||	         ||	         z   }t          |
|          }t          ||          }||k    r dS |                    |           |                    ||z
             |                    ||
z
             |                    ||z
             d||||fS )N)FNNNNT)rB   re   rt   r   rr   )desc_offset
desc_shapeshard_offsetshard_shapendimoverlap_offsetoverlap_shapedesc_startsshard_startsrj   desc_lodesc_hishard_loshard_hilohis                   r   get_overlap_regionr      s   {DNMKL4[[ + +a.a.:a=0??[^3(##(##88000b!!!R"W%%%2<(((BM****\IIr    c           
     p   t          | j        | j        |j        |j                  \  }}}}}	t          |j        |j        |j        |j                  \  }
}}}}|s|
s
J d            ||k    r|
J d            t          ||          }||k    sJ d| d|             t	          t          t          |                              }t          j        |j	        ||	d t          |	|          D                       }t          j        |j	        ||d t          ||          D                       }nt	          t          t          |                              }t          j        |j	        ||	d t          |	|          D                       }t          j        |j	        ||d	 t          ||          D                       }|Y|D ]V}t          |          \  }}|rt          j        ||          },t          |t                    rt          j        ||          }Wt          j        ||           d S )
Nzno overlap!z?only post transpose operation could make overlap shape mismatchzoverlap shape mismatch: z vs c                    g | ]
\  }}||z   S r#   r#   r7   r   os      r   r9   z(assign_sharded_slice.<locals>.<listcomp>1       IIIDAq!a%IIIr    axesstartsendsc                    g | ]
\  }}||z   S r#   r#   r   s      r   r9   z(assign_sharded_slice.<locals>.<listcomp>8  s     JJJDAq!a%JJJr    c                    g | ]
\  }}||z   S r#   r#   r   s      r   r9   z(assign_sharded_slice.<locals>.<listcomp>B  r   r    c                    g | ]
\  }}||z   S r#   r#   r   s      r   r9   z(assign_sharded_slice.<locals>.<listcomp>I  r   r    )r   rH   rG   r   r   re   rB   r   slicelocal_tensorr   is_list_string	transposerT   rW   castassign)src_desc	src_sharddst_desc	dst_shardpostprocess_listsrc_hasr8   r   src_desc_startssrc_shard_startsdst_hasoverlap_shape2dst_desc_startsdst_shard_startstransposed_src_overlap_shaper   src_tensor_slicedst_tensor_slicepsis_listr'   s                        r   assign_sharded_slicer   
  s    	" #!		
 	
 AGQ0@ 	" #!		
 	
 BGQ1A ,g,,},,&&++M ,++ (=+(
 (
$ ,~===Y'CYYYY >== E#:;;<<==!<"#IIC(8-$H$HIII	
 
 
 "<"#JJC(8.$I$IJJJ	
 
 
 E#m,,--..!<"#IIC(8-$H$HIII	
 
 
 "<"#IIC(8-$H$HIII	
 
 
 #" 	I 	IB,R00OGV I#)#34Df#M#M  b#&& I'-{3CR'H'H$
M"$455555r    c                    t          t                    }| D ]7}|                                D ] \  }}||                             |           !8t	          |          S r   )r   r   rV   extendrU   )list_of_dictsmergedinfor]   r^   s        r   merge_shard_info_listr   X  se    F    JJLL 	  	 DAq1IQ	 <<r    c           	         t          | j        t          | j                  t          | j                  t          | j                  t          | j        j                  	                    d          d                   S )NrS   r"   )r[   rG   r*   rH   dtype)
r   r[   rE   rG   r*   rH   rW   r   r   split)vals    r   build_shard_descr   `  sk    G#/**3+,,C-..#"())//44R8   r    c                r    	 t          j        |           }t          |t                    rd|fndS #  Y dS xY w)NT)FN)astliteral_evalrT   r   )r   r'   s     r   r   r   j  sD    !!$$!+FD!9!9Lf~~}L{{s   .1 6c                   | d}	 t          j        |t           j        t           j        z  t           j        z            }t          j        |           	 t           j                            |          rt           j                            |          dk    rat          j
        dt          j                     d           	 t           j                            |          rt          j        |           d S d S t          j        | |           t          j
        dt          j                     d           t           j                            |          rt          j        |           d S d S # t           j                            |          rt          j        |           w w xY w# t          $ r- t          j
        dt          j                     d           Y d S w xY w)Nz.lockr   zProcess z) found the metadata file already written.z- successfully wrote the metadata to the file.zT could not acquire the lock; another process is writing or has written the metadata.)ro   openO_CREATO_EXCLO_WRONLYcloseru   existsgetsizer	   r   getpidremover   saveFileExistsError)dataru   	lock_pathfds       r   write_to_file_if_emptyr  r  s   I
WY
RY 6 DEE
	%w~~d## (=(=(A(AUry{{UUU    w~~i(( %	)$$$$$% % Kd###KU29;;UUU   w~~i(( %	)$$$$$% %rw~~i(( %	)$$$$% 
 
 
xry{{xxx	
 	
 	
 	
 	
 	

s7   AF& A+E- 3F& 8>E- 63F& -6F##F& &3GGc                   t          t                    }|                                 D ]/\  }}t          |          }||                             |           0g }t
          j                                        dk    rdnd}|r/t
          j                            |t          |          |           nt          |          g}t          |          S )Nr   TF)r   r   rV   r   rr   r   r?   get_world_sizer|   rU   r   )sharded_state_dictr~   state_shard_infor[   r   descgathered_infor   s           r   build_global_state_shard_infor    s    "4((&,,.. + +S$$$$T****M)88::Q>>ttEH 1,,4 011=	
 	
 	
 	
 .//0 ///r    c                b   t          | t                    s
J d            i }| D ]}|                                D ]s\  }}||vrg ||<   t          |t                    r+|D ]'}|||         vr||                             |           (N|||         vr||                             |           t|S )Nz'The global_state_dict should be a list.)rT   r   rV   rr   )global_state_dict_metadataoutrc   r[   r   items         r   merge_state_dict_metadatar    s    0$77  1 7 C0 ) )
"((** 
	) 
	)HC#~~C#t$$ ) . .D3s8++C---. c#h&&HOOC(((
	) Jr    sharded_weightsr   c                @   d }t          j                  }j        fdt          |          D             }j        }| D ]Uj        fdt          |          D             }g }t          |          D ]Y}t	          |         |                   }	t          ||         ||                   }
|	|
k    r n|                    |	|
f           Zfdt          |          D             }fdt          |          D             }fdt          |          D             }fdt          |          D             }t          t          t          |                              }t          j
        j        |||          } |j        |||           WS )	Nc                    t          t          t          |                              }t          j        | |||          }t          j        ||           | S )Nr   )output)r   re   rB   r   r   r   )
dst_tensor
dst_startsdst_ends
src_tensorr   views         r   _assign_slicez7recover_shard_tensor_from_shards.<locals>._assign_slice  sY    E#j//**++|T*8
 
 
 	j....r    c                F    g | ]}j         |         j        |         z   S r#   rH   rG   )r7   rj   sws     r   r9   z4recover_shard_tensor_from_shards.<locals>.<listcomp>  s-    OOOa""1%q(99OOOr    c                F    g | ]}j         |         j        |         z   S r#   r  )r7   rj   r   s     r   r9   z4recover_shard_tensor_from_shards.<locals>.<listcomp>  s,    PPPqQ_Q'!-*::PPPr    c                8    g | ]\  }}|d          |         z
  S r6   r#   r7   rj   ols_glo_starts      r   r9   z4recover_shard_tensor_from_shards.<locals>.<listcomp>  s*    OOO51b1A.OOOr    c                8    g | ]\  }}|d          |         z
  S r   r#   r"  s      r   r9   z4recover_shard_tensor_from_shards.<locals>.<listcomp>  s*    MMMBbek!n,MMMr    c                8    g | ]\  }}|d          |         z
  S r6   r#   r7   rj   r#  sw_glo_starts      r   r9   z4recover_shard_tensor_from_shards.<locals>.<listcomp>  s7       ,1Ar1Q'  r    c                8    g | ]\  }}|d          |         z
  S r&  r#   r(  s      r   r9   z4recover_shard_tensor_from_shards.<locals>.<listcomp>  s*    OOO51br!u|A.OOOr    r   )rB   rH   re   rG   rt   r   rr   rC   r   r   r   r   )r  r  r  dims
sw_glo_endsw_shape	s_glo_endoverlaprj   ol_startol_ends_startss_ends	sw_startssw_endsr   srcr   r$  r)  s    `               @@@r    recover_shard_tensor_from_shardsr7    s      r  D#LOOOO5;;OOOJ~H D DoPPPPE$KKPPP	t 	D 	DA;q><?;;H1z!}55F6!!NNHf-....OOOOIg<N<NOOOHMMMM)G:L:LMMMF   5>w5G5G  I POOOIg<N<NOOOGc(mm,,--D,T(  C M"/9gsCCCIr    	ckpt_pathrW   c           
        dddddddddd		}t           j                                        d
k    }|rt           j                                        nd}t	          j        |           }d |D             }|r4g }||i}t           j                            |||           d |D             }nd|i}dd}	 |	|          }
|
|         }t          t                    }i }|D ].}t          j	        
                    | |          }t          |d          5 }|                                D ]}|                    |          }t          |                                          }|                                }||v sJ | d            ||         }t#          dt%          |          z  |||d          }t'          |dt%          |          z  d|          }||                             |           |||<   	 d d d            n# 1 swxY w Y   0|rTg }g }t           j                            |t+          |          |           t           j                            |||           nt+          |          g}|g}t          t                    }|D ]7}|                                D ] \  }}||                             |           !8d |                                D             }i }|D ]}|                    |           t1          ||          }d}t3          |t          j	        
                    | |                     |r!t           j                            |           d S d S )Nbfloat16uint8int8int16boolfloat16float32float64)	U16U8I8I16BOOLF16F32F64BF16r   r   c                <    g | ]}|                     d           |S )z.safetensors)endswith)r7   ry   s     r   r9   z+create_hf_ckpt_metadata.<locals>.<listcomp>  s9       T]]>-J-J  r    c                H    i | ]}|                                 D ]\  }}||	 S r#   )rV   )r7   dr   rx   s       r   
<dictcomp>z+create_hf_ckpt_metadata.<locals>.<dictcomp>  sG     
 
 
qwwyy
 
8CeD%
 
 
 
r    rank_visible_filesdict[int, list[str]]r0   c                z  	 t                      }|                                 D ]}|                    |           t          |          }t	          t                    |                                 D ]%\  }}|D ]}|                             |           &t	          t                    	|                    fd           t          	                    | d          |D ]I}|         }t          |fd          }	|                             |           |xx         dz  cc<   J	fd| D             S )Nc                2    t          |                    | fS r   )rB   )r   
file2rankss    r   <lambda>z?create_hf_ckpt_metadata.<locals>.assign_files.<locals>.<lambda>  s    c*Q-&8&8!%< r    )r[   r   c                    |          | fS r   r#   )r	rank_loads    r   rU  z?create_hf_ckpt_metadata.<locals>.assign_files.<locals>.<lambda>  s    ilA5F r    r   c                >    i | ]}|                     |g           S r#   )get)r7   r   r'   s     r   rO  zAcreate_hf_ckpt_metadata.<locals>.assign_files.<locals>.<dictcomp>  s)    JJJtfjjr**JJJr    )setr   updater   r   rV   rr   sortrU   fromkeysr   )
rP  	all_filesrx   r   r   
candidatesmin_rankrT  rX  r'   s
          @@@r   assign_filesz-create_hf_ckpt_metadata.<locals>.assign_files  sp    EE	'..00 	$ 	$EU####OO	 &&
-3355 	+ 	+KD% + +1$$T****+ T""<<<<===MM"4a88	 	% 	%A#AJ:+F+F+F+FGGGH8##A&&&h1$JJJJ7IJJJJr    r;   )	frameworkz is not supported yet.r6   F)rH   rG   r   r*   is_flattened)
tensor_keyrH   rd  rG   c                4    i | ]\  }}|t          |          S r#   )r   )r7   r]   r^   s      r   rO  z+create_hf_ckpt_metadata.<locals>.<dictcomp>K  s$    NNN$!Q1d1ggNNNr    )state_dict_metadatastorage_metadataz!flex-ckpt.auto_generated.metadata)rP  rQ  r0   rQ  )r   r?   r  r@   ro   rp   r|   r   r[  ru   rY   r   keys	get_slicerE   	get_shape	get_dtyper   rB   r   addrU   rV   r\  r   r  barrier) r8  r~   dtype_mappingr   cur_rankaccessible_filessafetensors_filesrP  local_filesrb  	rank2fileneed_handle_fileslocal_state_dict_metadatalocal_storage_metadata	file_name	file_pathr   r[   t_sr   r   ltmltir  global_storage_metadatarg  mdr]   r^   rh  metadataMETADATA_FILE_NAMEs                                    r   create_hf_ckpt_metadatar    s}   
 
 
M !0022Q6H08?v!**,,,aHz),, )    
4!23,,]	
 	
 	

 
!3
 
 
  !23K K K K6 /00I!(+ +C 0 0& 8 8	GLLI66	yD111 	8Qvvxx 8 8kk#&&cmmoo..---%/O/O/O---%e,)"&U"3 %!&!&   '""&U"3!& %	   *#.223777.7&s++)8	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8.  ;%'""$,,&*++	
 	
 	

 	,,#%;]	
 	
 	
 	
 '++D&E&E%F"#9":%c**( - -HHJJ 	- 	-DAq"))!,,,,	-NN2E2K2K2M2MNNN% $ $####/)  H
 ="',,y*<==    2""=111112 2s   C"G==H	H	c                   t           j                                        dk    }t          d | D                       r|j        |j        f}n|j        }| |         }|r(|                                r|                                n|S )Nr   c              3  @   K   | ]}t          |t                    V  d S r   )rT   rE   )r7   r]   s     r   r   z$get_target_tensor.<locals>.<genexpr>a  s,      
;
;A:a
;
;
;
;
;
;r    )r   r?   r  r   tensor_namedst_global_offsetis_dist_local_value)target_state_dict	read_itemr   r[   tensors        r   get_target_tensorr  _  s    !0022Q6H

;
;):
;
;
;;; $$i&AB#s#F$,M1A1AM6   vMr    c                    |s| j         r
J d            | S d t          ||          D             }t          t          | j                            }t          j        | |||          S )Nz5Only 0-dimensional tensor supports empty slice_shape.c                    g | ]
\  }}||z   S r#   r#   )r7   startlengths      r   r9   z slice_tensor.<locals>.<listcomp>q  s-       (5&  r    r   )r   r   r   re   r   r   r   )r  slice_beginslice_shape	slice_endr   s        r   slice_tensorr  j  s     < 	
 	
C	
 	
  ,/[,I,I  I fk""##D<T+INNNNr    c                   t          | t          j                  r|                                 sdS |                                 rk|                                 }| j        |_        t          | j                  dk    r t          | j        | j
        | j                  nd\  }}| j        }||dS nWt          | j                  }t          | j                  dk    r%t          dgt          | j                  z            nd}|}| }d}d }nRt          | t                    r+| j        }| j        }| j        }| j        }| j        }| j        }nt)          d|            t+          |j                                      d          d         }||j        |j        f}nd }t5          t          |          t          |          |t          |          ||          }|d u |d u k    s
J d	            ||fS )
Nr4   r   )r#   r#   r#   Fz<The value of state_dict should be a paddle.Tensor, but got: rS   r   zJlocal_tensor and local_tensor_metadata must both be None or both not None!)rT   r   rX   _is_initializedr  r  namerB   r   rP   r,   r.   rE   r   r   rG   rH   r*   rd  flattened_rangerZ   rW   r   r   r  stopr   )	r   r   rG   rH   r*   rd  r  local_tensor_dtypelocal_tensor_metadatas	            r   extract_tensor_metadatar  x  s5   #v}%% +
""$$ 	:;;== 	++--L #L sy>>A%% 6I$N    9L"m&;!z '<  	**K/239~~/A/AqcC	NN*+++r  'LL	C	'	' 

'o)''-P3PP
 
 	
 \/0066s;;A>"*0/2FG/mkl  D &;t&CDDDT EDD ...r    c           	        d}|rt           j                                        nd}| d}t          j                            | |          }t          j                            |          sd}i }	|                                D ]\  }
}t          |          \  }}|||	|
<   |r|	                                }d |
                    |g           D             }|	                                D ]6\  }
}t          |
|j        |j        |j        |j        d           }||vrd} n7|r2g }t           j                            |||           t#          |          S |S )NTr   z	_0.distcpFc                0    h | ]}t          |d           S )N)
replica_idr   )r7   indexs     r   	<setcomp>z*check_resumable_locally.<locals>.<setcomp>  s5     
 
 
 Ed+++
 
 
r    )re  rH   rd  r  rG   r  )r   r?   r@   ro   ru   rY   isfilerV   r  get_file_storage_inforZ  r   rH   rd  r  rG   r|   r   )ru   rc   metadata_managerr   r~   
local_loadr   checkpoint_filery  rg  r[   r\   r8   r  file_storage_infocur_file_storagelocal_tensor_indexglobal_local_loadss                     r   check_resumable_locallyr    s    J,4;6&&(((!D(((OT?33I7>>)$$ 
 &&(( = =
U#:5#A#A   ,'<$ ,BBDD
 
*..CC
 
 

 +>*C*C*E*E 	 	&C&!13A2? 5 E1=" " " ")999"
 :  ,,
M	
 	
 	
 %&&&r    )r   r   r   r   )r*   r+   r,   r-   r.   r/   r0   r1   )TNr   )r  r   )r8  rW   )8
__future__r   r   rA   ro   rm   collectionsr   dataclassesr   typingr   numpyr;   safetensors.numpyr   r   'paddle.distributed.fleet.utils.log_utilr	   aoa.aoa_enginer   r  r   r   r   sharded_weightr   r   paddle.frameworkr   r   r)   rP   ra   rk   rz   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r7  r  r  r  r  r  r#   r    r   <module>r     s   # " " " " " 



  				 				 # # # # # #                       ' ' ' ' ' '  : : : : : :              
       
  &%%%%%%	 	 	 	  #4 #4 #4 #4L' ' '6  (- - -G G G  $ $ $* * *&B B B* * * *:J J J0 @DK6 K6 K6 K6\      
 
 
00 0 0&  *' ' ' 'X @2 @2 @2 @2 @2FN N NO O O>/ >/ >/B, , , , ,r    