
    Αi^                       S SK Jr  S SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  S SKrS SKJr  S SKrS SKJr  SS	KJr  S
SKJrJrJr  S
SKJrJr  \(       a  S SKJr  S(S jrS r        S)S jr S r!S r"S r#S r$S r%S r&S r'S r(S*S jr)S r* S+S jr+S r,S r-S r.S r/S  r0S! r1S,S" jr2 S+ S-S# jjr3S$ r4S% r5S& r6S' r7g).    )annotationsN)defaultdict)replace)TYPE_CHECKING)	safe_open)logger   )postprocess_transpose   )LocalTensorIndexLocalTensorMetadataMetadata)ShardedWeightShardedWeightDesc)corec                    [         R                  " U 5      n X:H  R                  5       nUR                  S   S;   d   SUR                   35       eUR                  S   S:  a  US   R	                  5       $ S $ )Nr   )r   r   zrand_coordinator.shape: )paddle	to_tensornonzeroshapetolist)meshrankrand_coordinators      l/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/flex_checkpoint/dcp/utils.pyget_coordinatorr   /   s    D!D--/!!!$ )  ; 
""2"8"8!9:; 
 )9(>(>q(AA(E""$KO    c                b    X-  S:g  nX-   S-
  U-  /U-  nU(       a  US   nXDU-  U -
  -
  US'   U$ )Nr   r    )
total_numsnum_of_pieceshas_remainderresult
last_values        r   balanced_splitr&   <   sP    .!3M)A--?@=PFBZ
#=
#JKr
Mr   c                   SSK Jn  [        R                  " UR                  5      R                  UR                  5      n[        R                  R                  5       U;  a  g[        U[        R                  R                  5       5      n[        R                  " U 5      nU  Vs/ s H  nSPM     nnU" U[        U 5      5      u  p[        U	5       H  u  p[        U5      S:X  a  M  S/[        U 5      -  nU HX  nX]   n[        Xj   UR                  U   5      nX   Xj'   [        US U 5      X'   X   X   ::  a  X   X'   MJ  X==   X   -  ss'   MZ     M     [!        U5      [!        U5      4$ s  snf )Nr   )placemetns_to_dist_statusNN)/paddle.distributed.auto_parallel.placement_typer(   nparrayprocess_idsreshaper   r   distributedget_rankr   copylen	enumerater&   sumtuple)global_shapeprocess_mesh
placementsr(   r   rank_coordinatorlocal_shape_global_offsetdims_mapping
tensor_dim	mesh_dimslocal_offsetmesh_dim	chunk_idxchunkss                   r   %compute_local_shape_and_global_offsetrD   E   sa   
 88L,,-55l6H6HID""$D0&tV-?-?-H-H-JK))L)K ,-1QM-/
C<MNOL!*<!8
y>QsS..!H(2I#'););H)EF '-&7K#'*6*9+='>L$(L,DD,8,D))\-EE) "	 "9" u]333) .s   %E/c                :   ^^^ 0 m0 mUUU4S jmT" SU 5        TT4$ )zS
Flatten the nested dict to a flat dict.
{"model": {"w0": xxx}} -> {model.w0: xxx}
c                Z  > [        U[        5      (       aG  UR                  5        H2  u  p#[        U[        5      (       d
   SU 35       eT" / U QUP7U5        M4     g [        U[        R
                  [        45      (       a  SR                  U 5      nUTU'   U TU'   g [        SU 35      e)NzThe key should be str, but is .z2The value should be dict or paddle.Tensor, but is )	
isinstancedictitemsstrr   Tensorr   join
ValueError)keyvaluekvflatten_key_str_flattenflatten_state_dictmappings        r   rT   $flatten_state_dict.<locals>._flattens   s    eT""!!S))O-KA3+OO)3A& & }=>>!hhsmO27/'*GO$DUGL r   r    r    )
state_dictrT   rU   rV   s    @@@r   rU   rU   k   s,    
 G Rw&&r   c                    0 nU R                  5        He  u  p4X   n[        U[        5      (       d
   SU 35       eUn[        [	        U5      S-
  5       H  nXW   nUR                  U0 5      nM     XFUS   '   Mg     U$ )zU
Unflatten the flat dict to a nested dict.
{model.w0: xxx} -> {"model": {"w0": xxx}}
z The key should be tuple, but is r   r   )rJ   rH   r5   ranger2   
setdefault)flat_state_dictrV   rX   rO   rP   	key_tupletmpis           r   unflatten_state_dictr`      s    
 J%++-
L	)U++ 	
.yk:	
+ s9~)*A,C..b)C + #IbM . r   c                    S/n[         R                  " S5      n[        R                  " U 5      nU HF  nUR	                  U5      nU(       d  M  UR                  [        UR                  S5      5      5        MH     U(       a  [        U5      $ S $ )Nr   z^(\d+)_(\d+)\.distcp$r	   )	recompileoslistdirmatchappendintgroupmax)pathnumberspatternfilesfilerf   s         r   
get_max_idrp      so    cGjj12GJJtEd#5NN3u{{1~./  #3w<,,r   c                    / n[         R                  R                  X U5        USS   H  nX2S   :X  a  M   SU S35       e   g )Nr   r   zid:z !=  all_unique_id[0])r   r/   all_gather_object)	unique_idprocess_groupall_unique_idids       r   check_unique_idrw      sS    M
((- AB1%%FRD0E'FF%  r   c                @    Sn[        X5       H  u  p4X$-  U-   nM     U$ )Nr   )zip)indicesr   idxr_   dims        r   ravel_indexr}      s(    
Cg%i!m &Jr   c                    / n[        U5       H  nUR                  X-  5        X-  n M     [        [        U5      5      $ N)reversedrg   r5   )r{   r   rz   r|   s       r   unravel_indexr      s<    Gsy!  '"##r   c                4   [        X5      n[        US-
  U 5      n/ n[        [        U 5      5       Ha  nUS:X  a  X6   nXF   S-   nO:X6S-
     XFS-
     :X  a$  [        X6   XF   5      n[	        X6   XF   5      S-   nOSnX   nUR                  Xx45        Mc     XSU4$ )Nr   r   )r   rZ   r2   minrj   rg   )	r   
flat_startflat_end	start_idxend_idx
min_slicesaxisses	            r   minimal_nd_slicer      s    j0IHqL%0GJc%j!19A!A"gQh&77	7	7!;K1&! " '))r   c                L    [        S U 5       5      n[        X@5      nX%-
  X5-
  4$ )Nc              3  *   #    U  H	  oS    v   M     g7fr   Nr    ).0r   s     r   	<genexpr>*flat_range_in_min_slice.<locals>.<genexpr>   s     0ZtZ   )r5   r}   )r   r   r   r   
min_startsmin_flat_starts         r   flat_range_in_min_slicer      s-    0Z00J 3N&(AAAr   c                   [        U R                  5       5      n[        S U 5       5      n[        S U 5       5      nU(       a  U(       d  [	        S5      eU(       d  U$ U(       a  SnO[        U5      S:X  a  SnOSn/ n[        R                  R                  XvU5        [        S U 5       5      (       d  [        S U 5       5      (       d   S	5       e[        S
 U 5       5      $ )Nc              3  B   #    U  H  n[        U[        5      v   M     g 7fr   rH   r   r   rR   s     r   r   (is_sharded_state_dict.<locals>.<genexpr>   s     Fv!A}55v   c              3  B   #    U  H  n[        U[        5      v   M     g 7fr   r   r   s     r   r   r      s     CFqjM22Fr   z?All values must be ShardedWeight if any value is ShardedWeight.r   r   r   c              3  *   #    U  H	  oS :  v   M     g7fr   r    r   fs     r   r   r           )y!Avyr   c              3  *   #    U  H	  oS :*  v   M     g7fr   r    r   s     r   r   r      s     0KAar   zUNot support mixed type of ShardedWeight and non-ShardedWeight in the same state_dict!c              3  *   #    U  H	  oS :  v   M     g7fr   r    r   s     r   r   r      r   r   )	listvaluesallany	TypeErrorr2   r   r/   rr   )rX   use_distrt   r   is_all_shardedhas_shardedflag	all_flagss           r   is_sharded_state_dictr      s    *##%&FFvFFNCFCCK>M
 	
 	V	I
((-H)y)))S0K0K-K-K _K )y)))r   c                X   [        U 5      n/ n/ n/ n/ n[        U5       H  n	X	   n
X	   X   -   nX)   nX)   X9   -   n[        X5      n[        X5      nX:  a    gUR	                  U5        UR	                  X-
  5        UR	                  X-
  5        UR	                  X-
  5        M     SXVXx4$ )N)FNNNNT)r2   rZ   rj   r   rg   )desc_offset
desc_shapeshard_offsetshard_shapendimoverlap_offsetoverlap_shapedesc_startsshard_startsr_   desc_lodesc_hishard_loshard_hilohis                   r   get_overlap_regionr      s    {DNMKL4[..:=0??[^3##80b!RW%2<(BM*  IIr   c                *   [        U R                  U R                  UR                  UR                  5      u  pVpxn	[        UR                  UR                  UR                  UR                  5      u  ppnU(       d  U
(       d   S5       eX{:w  a  Uc   S5       e[        Xt5      nX:X  d   SU SU 35       e[	        [        [        U5      5      5      n[        R                  " UR                  UU	[        X5       VVs/ s H  u  nnUU-   PM     snnS9n[        R                  " UR                  UU[        X5       VVs/ s H  u  nnUU-   PM     snnS9nO[	        [        [        U5      5      5      n[        R                  " UR                  UU	[        X5       VVs/ s H  u  nnUU-   PM     snnS9n[        R                  " UR                  UU[        X5       VVs/ s H  u  nnUU-   PM     snnS9nUbe  U H_  n[        U5      u  nnU(       a  [        R                  " UU5      nM1  [        U[        5      (       d  MH  [        R                  " UU5      nMa     [        R                   " UU5        g s  snnf s  snnf s  snnf s  snnf )Nzno overlap!z?only post transpose operation could make overlap shape mismatchzoverlap shape mismatch: z vs axesstartsends)r   r<   r:   r
   r   rZ   r2   r   slicelocal_tensorry   is_list_string	transposerH   rK   castassign)src_desc	src_sharddst_desc	dst_shardpostprocess_listsrc_hasr;   r   src_desc_startssrc_shard_startsdst_hasoverlap_shape2dst_desc_startsdst_shard_startstransposed_src_overlap_shaper   r   osrc_tensor_slicedst_tensor_slicepsis_listr$   s                          r   assign_sharded_slicer   
  s    	""  ##!!		
 AG0@ 	""  ##!!		
 BG1A g,},&+ 	
M	
+ (=(
$ ,= 	
&'C&DDHXY	
= E#:;<=!<<""#$'(8$HI$HDAq!a%$HI	
 "<<""#$'(8$IJ$IDAq!a%$IJ	
 E#m,-.!<<""#$'(8$HI$HDAq!a%$HI	
 "<<""#$'(8$HI$HDAq!a%$HI	
 #"B,R0OGV#)#3#34Df#M b#&&'-{{3CR'H$ # MM"$45I J K J Js   >I=J"J	%Jc                    [        [        5      nU  H/  nUR                  5        H  u  p4X   R                  U5        M     M1     [	        U5      $ r   )r   r   rJ   extendrI   )list_of_dictsmergedinforQ   rR   s        r   merge_shard_info_listr   X  sD    FJJLDAIQ !  <r   c           	         [        U R                  [        U R                  5      [        U R                  5      [        U R
                  5      [        U R                  R                  5      R                  S5      S   S9$ )NrG   r   )rO   r:   r6   r<   dtype)
r   rO   r5   r:   r6   r<   rK   r   r   split)vals    r   build_shard_descr   `  sb    GG#//*3++,C--.#""(()//4R8 r   c                t     [         R                  " U 5      n[        U[        5      (       a  SU4$ S$ !    g= f)NT)FN)astliteral_evalrH   r   )r   r$   s     r   r   r   j  s<    !!!$!+FD!9!9f~L}Ls   .3 3 7c                $   U S3n [         R                  " U[         R                  [         R                  -  [         R                  -  5      n[         R
                  " U5         [         R                  R                  U5      (       a  [         R                  R                  U5      S:  aj  [        R                  " S[         R                  " 5        S35         [         R                  R                  U5      (       a  [         R                  " U5        g g [        R                  " X5        [        R                  " S[         R                  " 5        S35        [         R                  R                  U5      (       a  [         R                  " U5        g g ! [         R                  R                  U5      (       a  [         R                  " U5        f f = f! [         a0    [        R                  " S[         R                  " 5        S35         g f = f)Nz.lockr   zProcess z) found the metadata file already written.z- successfully wrote the metadata to the file.zT could not acquire the lock; another process is writing or has written the metadata.)rd   openO_CREATO_EXCLO_WRONLYcloserk   existsgetsizer   r   getpidremover   saveFileExistsError)datark   	lock_pathfds       r   write_to_file_if_emptyr   r  s^   &I
WWY

RYY 6 DE
	%ww~~d##(=(Aryy{m+TU  ww~~i((		)$ ) KK#KK299;-'TU ww~~i((		)$ )rww~~i((		)$ ) 
ryy{m#wx	

s8   AG %A4F :G AF :G =GG 7HHc                p   [        [        5      nU R                  5        H#  u  p4[        U5      nX#   R	                  U5        M%     / n[
        R                  R                  5       S:  a  SOSnU(       a+  [
        R                  R                  U[        U5      U5        O[        U5      /n[        U5      $ )Nr   TF)r   r   rJ   r   rg   r   r/   get_world_sizerr   rI   r   )sharded_state_dictrt   state_shard_inforO   r   descgathered_infor   s           r   build_global_state_shard_infor    s    "4(&,,.$$$T* / M))88:Q>tEH,,4 01=	
 ./0 //r   c                L   [        U [        5      (       d   S5       e0 nU  H  nUR                  5        Hh  u  p4X1;  a  / X'   [        U[        5      (       a(  U H   nXQU   ;  d  M  X   R                  U5        M"     MK  XAU   ;  d  MU  X   R                  U5        Mj     M     U$ )Nz'The global_state_dict should be a list.)rH   r   rJ   rg   )global_state_dict_metadataoutrX   rO   r   items         r   merge_state_dict_metadatar    s    0$77 17 C0
"((*HC~#t$$Ds8+-   #h&HOOC( + 1 Jr   c                .   S n[        UR                  5      nUR                  n[        U5       Vs/ s H!  oQR                  U   UR                  U   -   PM#     nnUR                  nU  GH  nUR                  n	[        U5       Vs/ s H!  oXR                  U   UR                  U   -   PM#     n
n/ n[        U5       H=  n[	        X   XE   5      n[        X   Xe   5      nX:  a    M  UR                  X45        M?     [        U5       VVs/ s H  u  p^US   X   -
  PM     nnn[        U5       VVs/ s H  u  p^US   X   -
  PM     nnn[        U5       VVs/ s H  u  p^US   XE   -
  PM     nnn[        U5       VVs/ s H  u  p^US   XE   -
  PM     nnn[        [        [        U5      5      5      n[        R                  " UR                  UUUS9nU" UR                  UUU5        GM     U$ s  snf s  snf s  snnf s  snnf s  snnf s  snnf )Nc                    [        [        [        U5      5      5      n[        R                  " XXS9n[        R
                  " X5S9  U $ )Nr   )output)r   rZ   r2   r   r   r   )
dst_tensor
dst_startsdst_ends
src_tensorr   views         r   _assign_slice7recover_shard_tensor_from_shards.<locals>._assign_slice  s<    E#j/*+||*
 	j.r   r   r   r   )r2   r<   rZ   r:   rj   r   rg   r3   r   r   r   r   )sharded_weightsswr  dimssw_glo_startr_   
sw_glo_endsw_shaper   s_glo_start	s_glo_endoverlapol_startol_endols_startss_ends	sw_startssw_endsr   srcs                        r    recover_shard_tensor_from_shardsr#    s    r D##LCH;O;a""1%q(99;JO~~HooDI$KPKq__Q'!--*::K	PtA;><?;Hz}5F!NNH-.  =Fg<NO<N511.<NHO:CG:LM:Lbekn,:LFM5>w5G5GEA1'5G   =Fg<NO<N51r!u|.<NGOc(m,-D,,T(C "//9gsC/ 2 I9 P
 Q PM Ps#   (G5(G:G?4HHHc                    SSSSSSSSSS	.	n[         R                  R                  5       S
:  nU(       a  [         R                  R                  5       OSn[        R
                  " U 5      nU Vs/ s H  ofR                  S5      (       d  M  UPM     nnU(       aU  / nXG0n	[         R                  R                  XU5        U V
VVs0 s H  oR                  5         H  u  pX_M	     M      nnn
nOSU0n    SS jnU" U5      nX   n[        [        5      n0 nU H  n[        R                  R                  U U5      n[        USS9 nUR                  5        H  nUR                  U5      n[!        UR#                  5       5      nUR%                  5       nUU;   d
   U S35       eUU   n['        S[)        U5      -  UUUSS9n[+        US[)        U5      -  SUS9nUU   R-                  U5        UUU'   M     S S S 5        M     U(       aP  / n/ n[         R                  R                  U[/        U5      U5        [         R                  R                  UUU5        O[/        U5      /nU/n[        [        5      nU H1  nUR                  5        H  u  nn UU   R1                  U 5        M     M3     UR                  5        VV s0 s H  u  nn U[3        U 5      _M     nnn 0 n!U H  nU!R1                  U5        M     [5        UU!S9n"Sn#[7        U"[        R                  R                  U U#5      5        U(       a   [         R                  R9                  U5        g g s  snf s  snnn
f ! , (       d  f       GM[  = fs  sn nf )Nbfloat16uint8int8int16boolfloat16float32float64)	U16U8I8I16BOOLF16F32F64BF16r   r   z.safetensorsc                B  ^^	 [        5       nU R                  5        H  nUR                  U5        M     [        U5      n[	        [        5      mU R                  5        H"  u  p2U H  nTU   R                  U5        M     M$     [	        [        5      nUR                  U4S jS9  [        R                  U S5      m	U H6  nTU   n[        UU	4S jS9nXW   R                  U5        T	U==   S-  ss'   M8     U  Vs0 s H  o3UR                  U/ 5      _M     sn$ s  snf )Nc                $   > [        TU    5      U 4$ r   )r2   )r   
file2rankss    r   <lambda>?create_hf_ckpt_metadata.<locals>.assign_files.<locals>.<lambda>  s    c*Q-&8!%<r   )rO   r   c                   > TU    U 4$ r   r    )r	rank_loads    r   r9  r:    s    ilA5Fr   r   )setr   updater   r   rJ   rg   sortrI   fromkeysr   get)
rank_visible_files	all_filesrn   r   r   r$   
candidatesmin_rankr8  r=  s
           @@r   assign_files-create_hf_ckpt_metadata.<locals>.assign_files  s	    E	'..0EU# 1O	 &
-335KD1$$T*  6 T"<=MM"4a8	A#AJ:+FGH##A&h1$	  8JJ7Itfjjr**7IJJJs   =Dr+   )	frameworkz is not supported yet.)r   F)r<   r:   r   r6   is_flattened)
tensor_keyr<   rJ  r:   )state_dict_metadatastorage_metadataz!flex-ckpt.auto_generated.metadata)rC  dict[int, list[str]]returnrN  )r   r/   r   r0   rd   re   endswithrr   rJ   r   r>  rk   rM   r   keys	get_slicer5   	get_shape	get_dtyper   r2   r   addrI   r?  r   r   r   barrier)$	ckpt_pathrt   dtype_mappingr   cur_rankaccessible_filesro   safetensors_filesrC  local_filesdr   rn   rG  	rank2fileneed_handle_fileslocal_state_dict_metadatalocal_storage_metadata	file_name	file_pathr   rO   t_sr   r   ltmltir  global_storage_metadatarL  mdrQ   rR   rM  metadataMETADATA_FILE_NAMEs$                                       r   create_hf_ckpt_metadatark    s   
 
M !!002Q6H08v!!**,aHzz),))]]>-J)   3,,]	
 "4
!3AwwyDKyD!3 	 
  !23K0K	K6 /0I!+ +C 0&	GGLLI6	yD1Qvvxkk#&cmmo.-O%8N/OO-%e,)"&U"3 %!&!& '""&U"3!& %	 *#.2237.7&s+)   21 '2 %'""$,,&*+	

 	,,#%;]	
 '++D&E%F"#9":%c*(HHJDAq"))!,  ) 3F2K2K2MN2M$!Q1d1g:2MN%# & /)H
 ="'',,y*<= ""=1 W
P 21T Os%   /MM%M 	B1M'M:'
M7	c                    [         R                  R                  5       S:  n[        S U  5       5      (       a  UR                  UR
                  4nOUR                  nX   nU(       a%  UR                  5       (       a  UR                  5       $ U$ )Nr   c              3  B   #    U  H  n[        U[        5      v   M     g 7fr   )rH   r5   )r   rQ   s     r   r   $get_target_tensor.<locals>.<genexpr>a  s     
;):A:a):r   )r   r/   r   r   tensor_namedst_global_offsetis_dist_local_value)target_state_dict	read_itemr   rO   tensors        r   get_target_tensorrv  _  sx    !!002Q6H

;):
;;;$$i&A&AB###F$,1A1A6 MvMr   c                    U(       d  U R                   (       a   S5       eU $ [        X5       VVs/ s H	  u  p4X4-   PM     nnn[        [        U R                  5      5      n[
        R                  " XXS9$ s  snnf )Nz5Only 0-dimensional tensor supports empty slice_shape.r   )r   ry   r   rZ   r   r   r   )ru  slice_beginslice_shapestartlength	slice_endr   s          r   slice_tensorr}  j  sv    << 	
C	
  -0,I,I=5,I   fkk"#D<<+NN	s   A6c                *   [        U [        R                  5      (       Ga  U R                  5       (       d  gU R	                  5       (       a~  U R                  5       nU R                  Ul        [        U R                  5      S:  a+  [        U R                  U R                  U R                  5      OSu  nnU R                  nUb  Uc  gOV[        U R                  5      n[        U R                  5      S:  a"  [        S/[        U R                  5      -  5      OSnUnU nSnS nOl[        U [        5      (       aI  U R                  nU R                  nU R                   nU R"                  nU R$                  nU R&                  nO[)        SU  35      e[+        UR,                  5      R/                  S5      S   nUb  UR0                  UR2                  4nOS n[5        [        U5      [        U5      U[        U5      UU5      nUS L US L :X  d   S	5       eX4$ )
Nr)   r   )r    r    r    Fz<The value of state_dict should be a paddle.Tensor, but got: rG   r   zJlocal_tensor and local_tensor_metadata must both be None or both not None!)rH   r   rL   _is_initializedrq  rr  namer2   r   rD   r7   r8   r5   r   r   r:   r<   r6   rJ  flattened_rangerN   rK   r   r   rz  stopr   )	r   r   r:   r<   r6   rJ  r  local_tensor_dtypelocal_tensor_metadatas	            r   extract_tensor_metadatar  x  s   #v}}%%""$$;;==++-L #L syy>A% 6II$$NN  99L"m&;! '<  		*K/2399~/AqcC		N*+r  'LL	C	'	'''oo))''''--J3%P
 	
 \//066s;A>"*00/2F2FG/mkl D &;t&CD TD ..r   c           
        SnU(       a  [         R                  R                  5       OSnU S3n[        R                  R                  X5      n[        R                  R                  U5      (       d  Sn0 n	UR                  5        H  u  p[        U5      u  pUc  M  XU
'   M     U(       a  UR                  5       nUR                  U/ 5       Vs1 s H  n[        US S9iM     nnU	R                  5        HE  u  p[        U
UR                  UR                  UR                  UR                   S S9nUU;  d  MC  Sn  O   U(       a-  / n[         R                  R#                  UXT5        [%        U5      $ U$ s  snf )NTr   z	_0.distcpF)
replica_id)rK  r<   rJ  r  r:   r  )r   r/   r0   rd   rk   rM   isfilerJ   r  get_file_storage_inforB  r   r   r<   rJ  r  r:   rr   r   )rk   rX   metadata_managerr   rt   
local_loadr   checkpoint_filerc  rL  rO   rP   r;   r  file_storage_infoindexcur_file_storagelocal_tensor_indexglobal_local_loadss                      r   check_resumable_locallyr    su    J,46&&(!Di(OT3I77>>)$$
 &&(
#:5#A  ,'<$ )
 ,BBD +..C
C Ed+C 	 

 +>*C*C*E&C!13AA2?? 5 E E1==" ")99"
 +F ,,
	
 %&&3
s   E5)r   znp.array | list[list[int]]r   rh   )r6   z	list[int]r7   zcore.ProcessMeshr8   zlist[core.Placement]rO  ztuple[tuple[int], tuple[int]])TNr   )r  r   )rW  rK   )8
__future__r   r   r1   rd   rb   collectionsr   dataclassesr   typingr   numpyr+   safetensors.numpyr   r   'paddle.distributed.fleet.utils.log_utilr   aoa.aoa_enginer
   ri  r   r   r   sharded_weightr   r   paddle.frameworkr   r   r&   rD   rU   r`   rp   rw   r}   r   r   r   r   r   r   r   r   r   r   r  r  r#  rk  rv  r}  r  r  r    r   r   <module>r     s   # 
  	 	 #     '  : 

 %	#4#4"#4 %#4 #	#4L'6(-G$*&B*:J0 @DK6\
00&*'X @2@2FNO>/B,r   