u-jO: ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z m Z ddl ZddlZddlmZddlmZddlmZdefdZdedefdZd Zd0d ed eddfd Zd eddfdZd eddfdZd1d ede eddfdZ d1de eeffdZ!dZ"dZ#dZ$dZ%dZ&d1dZ'd1dZ(dZ)dZ*dZ+ej,dZ-dZ.dZ/d1dZ0d ej1j2defd!Z3d"e d#ej4fd$ed%edej4fd&Z5d'e6ed(ed)ed*e7ede8e7eej4ff d+Z9 d2d,ej1j:d-ej;d.edej1j:fd/Zz _is_free_port..#s8??R1<<T ++q0??????r)r gethostbyname_ex gethostnameappendrrall)ripsrs` @r _is_free_portr&s  !&"4"6"6 7 7 ;CJJ{ v~v'9 : :@a?????3?????@@@@@@@@@@@@@@@@@@s,BBBc t|dr|jdi|}ndtjtjtjdkrt j|fi|}nt dtjd|S)Ncompilez 2.0.0.devzDCompiling model needs torch version > 2.0.0, your torch version is: z , origin model will be returned.)hasattrr(rparsetorch __version__print)modelcompile_optionss r compile_modelr1&sui    0000 u( ) )W];-G-G G G e7777  Z&+&7 Z Z Z    Lrnccllauncherbackendc tjdtjd|dkrt|fi|dS|dkrt |fi|dS|dkrt |fi|dSt d|)NT) allow_nonespawnpytorchmpislurmzInvalid launcher type: )mpget_start_methodset_start_method_init_dist_pytorch_init_dist_mpi_init_dist_slurm ValueError)r3r4kwargss r init_distrC4s d+++3 G$$$97--f----- U  w))&))))) W  ++F+++++=8==>>>rc ttjd}tj|t jdd|i|dS)N LOCAL_RANKr4r))intosenvironr,cuda set_devicedistinit_process_groupr4rB local_ranks rr>r>AsNRZ -..J J*%%%66G6v66666rc ttjd}tj|dtjvrdtjd<dtjvrt dtjdtjd<tjdtjd <tjd d |i|dS) NOMPI_COMM_WORLD_LOCAL_RANK MASTER_PORT29500 MASTER_ADDRz/The environment variable MASTER_ADDR is not setOMPI_COMM_WORLD_SIZE WORLD_SIZEOMPI_COMM_WORLD_RANKRANKr4r)) rFrGrHr,rIrJKeyErrorrKrLrMs rr?r?HsRZ <=>>J J*%%%BJ&&$+ =!BJ&&HIII!z*@ABJ|$:;BJv66G6v66666rcRttjd}ttjd}tjd}tj}tj||ztjd|d}|t|tjd<nVdtjvrnGtdrd tjd<n(tttjd<d tjvr|tjd <t|tjd <t||ztjd <t|tjd <tj |dS)aInitialize slurm distributed training environment. If argument ``port`` is not specified, then the master port will be system environment variable ``MASTER_PORT``. If ``MASTER_PORT`` is not in system environment variable, then a default port ``29500`` will be used. Args: backend (str): Backend of torch.distributed. port (int, optional): Master port. Defaults to None. SLURM_PROCID SLURM_NTASKSSLURM_NODELISTzscontrol show hostname z | head -n1NrQi(>BJ} %BJ&&$( =!"6{{BJ|"7X#566BJ|WBJvG,,,,,,rctrUddlm}|$|rddlm}|}t j|}t j|}nd}d}||fS)zGet dist info of a specified group Args: group: The parallel group, default None, for the global group Returns: A tuple of the current rank and world_size of the group r)is_megatron_initializedN)mpur ) is_distmodelscope.utils.megatron_utilsrh megatron_utilriget_data_parallel_grouprKget_rankget_world_size)grouprhrirank world_sizes r get_dist_infors|syy KKKKKK =4466= ) ) ) ) ) )//11E}U##(//   rc\ttjddS)NrEr)rFrGrHgetr)rrget_local_rankrvs rz~~lA.. / //rc|tjsdStjsdStjS)Nr)rK is_availableis_initializedrnr)rrrnrns:    q   q =??rc|tjsdStjsdStjS)Nr )rKrxryror)rrroros=    q   q    rctjsdStjsdStj}|dkrdStjdS)zj Helper function to synchronize (barrier) among all processes when using distributed training Nr )rKrxryrobarrier)rrs r synchronizer}s[       $&&JQLNNNNNrcNtjotjSN)rKrxryr)rrrjrjs     84#6#8#88rcRtrtj|dkndS)NrT)rjrKrnrps r is_masterrs%(/ ;4=  1 $ $t;rc0dtdtffd }|S)Nfuncr cJtjfd}|S)Nc6tr|i|SdSr)r)argsrBrrps rwrapperz.master_only..decorate..wrappers2 -tT,V,,, - -r) functoolswraps)rrrps` rdecoratezmaster_only..decorates>    - - - - -   -r)r)rprs` r master_onlyrs6xH OrctstjSd}trtj}t jt |d}|S)zRMake sure each rank has the same temporary directory on the distributed mode. Nr)rjtempfilemkdtemprrKr| broadcast)tmpdirs r make_tmp_dirrs] 99"!!! F{{$!##LNNN vq ! !F Mrctj}tjdgd}||kr[tjt t j|tjd}tj|jd}tj tj ||||kr5tj | fdtjd}tj tj ||t j |S)z Broadcasts the inputs to all ranks. Arguments: inputs : Any objects that can be serialized by pickle. src (int): Source rank. Returns: Each rank returns the same value as src. rrIdevicedtyper)rKrnr,tensor bytearraypickledumpsuint8shaper|rfullitemloadscpunumpytobytes)inputssrcrq shape_tensor inputs_tensors rrrs$ =??D<F333L s{{ fl6** + +5;vOOO |M$7GGG LNNNN<%%% s{{ L$5$5$7$7#:#$).*0222  LNNNN=#&&& < ))++1133;;== > >>rc|n|dkrhtj|tj|tj|tj|dStd|)Nrz0Random seed should be positive, current seed is )randomseednpr, manual_seedrImanual_seed_allrA)rs rset_random_seedrs DAII D t $ ""4((((( Et E EGG Grc|tjdkrtjdStjjS)zj Return a process group based on gloo backend, containing all the ranks The result is cached. r2gloor])rK get_backend new_grouprpWORLDr)rr_get_global_gloo_grouprs6  V##~f----zrctj|}|dvsJtj|dkrdnd}t j|}t |dkrKtd tt |dz |tj |}tj ||}|S)N)rr2rrrIi@z;Rank {} trying to all-gather {:.2f} GB of data on device {}r)rKrr,rrrlenloggerwarningformatrn ByteStorage from_buffer ByteTensorto)datarpr4rbufferstoragers r_serialize_to_tensorrsu%%G & & & & & \7f#4#4%%& A AF \$  F 6{{W I F8::v;;'*F 4 4 5 5 5++F33G  g & & ) ) ) 8 8F Mrctj|}|dks Jdtjgtjj}fdt|D}tj|||d|D}t|}||krBtj ||z ftj j}tj |fd|fS) zz Returns: list[int]: size of the tensor, on each rank Tensor: padded tensor that has the max size rr zBcomm.gather/all_gather must be called from ranks within the group!rc^g|])}tjdgtjj*S)r r)r,zerosint64r)r_rs r z*_pad_to_largest_tensor..0s@   QCu{6=AAArcPg|]#}t|$Sr))rFr)rsizes rrz*_pad_to_largest_tensor..5s(888dTYY[[!!888rrdim) rKror,rnumelrrrange all_gathermaxrrcat)rrprr local_size size_listmax_sizepaddings` r_pad_to_largest_tensorr#s  $5111JaK v||~~.$)K%+]444Jz""I OIz777788i888I9~~HX+x*47$)K%+]444FG,!444 f rc8tdkr|gS|t}tj|dkr|gSt||t |\}t |fd|D}tj||g}t||D]l\} d|}| tj |m|S)a; Run all_gather on arbitrary picklable data (not necessarily tensors). Args: data: any picklable object group: a torch process group. By default, will use a group which contains all ranks on gloo backend. Returns: list[data]: list of data gathered from each rank r Nc^g|])}tjftjj*S)r)r,emptyrr)rrrrs rrzall_gather..Zs@   XL FMJJJrr)rorrKrrrrziprrrr#rr) rrpr tensor_list data_listrrrrs @@rrrCs61v  }&(( 5!!Q&&v !$ . .F.vu==Iv9~~HK OKu5555II{33// f##%%--//6f--.... rr/ctd|Ddhz }t|dkS)Nc3>K|]}t|jVdSr)rar)rps rr z$is_on_same_device..is*??qS]]??????rrr )set parametersr)r/ device_sets ris_on_same_devicerhsB??E,<,<,>,>?????5'IJ z??a r forward_fn. chunk_size chunk_dimc t|dks J|dttjj}|t|kr#t d|dt|d|dkr|dj}|D]3}|j|kr t d|d|j4|dj|zdkr&t d|djd ||dj|z t  fd |D}t fd t|D}tj | S|S) Nrz" has to be a tuple/list of tensorszforward_chunk_fn expects z arguments, but only z input tensors are givenz/All input tenors have to be of the same shape: z, found shape zThe dimension to be chunked z( has to be a multiple of the chunk size c3FK|]}|VdS)rN)chunk)r input_tensorr num_chunkss rr z,apply_chunking_to_forward..sJ%/%/   zy  9 9%/%/%/%/%/%/rc3"K|] }|V dSrr))rinput_tensors_chunkrs rr z,apply_chunking_to_forward..sGCC# J+ ,CCCCCCrr) rinspect signaturerrArtuplerr,r) rrr input_tensorsnum_args_in_forward_chunk_fn tensor_shaperinput_tensors_chunks output_chunksrs ` ` @rapply_chunking_to_forwardrmsZ }     &JJJ   $'*%%0$2$2 #s='9'999 (D  [^_l[m[m   !! !A~~$Q'-i8 ) D DL!), << ClCC#/#5i#@CCDDD=   !) ,z 9Q > >%}Q/?/Ei/P%%"%%&& &#1%+I6*D  %%/%/%/%/%/ -%/%/%/ / /CCCC'*,@'ACCCCC yI6666 :} %%rheadsn_heads head_sizealready_pruned_headsctj||}t||z }|D]%tfd|Dz d|<&|dd}tjt|| }||fS)Nc3,K|]}|krdndVdS)r rNr))rhheads rr z3find_pruneable_heads_and_indices..s/MM1q4xx!!QMMMMMMrrrr ) r,onesrsumview contiguouseqarangerlong)rrrrmaskindexrs @r find_pruneable_heads_and_indicesrs :gy ) )D     EcMMMM8LMMMMMMT 99R== # # % % ( ( + +D#l3t9955d;@@BBE %<rlayerrrc||jj}|j||}|jc|dkr,|j}n1|j|}t|j}t|||<tj |d|d|jdu|jj}d|j_ |j|d|j_ |jDd|j_ |j|d|j_ |S)Nr r)biasFT)rweightr index_selectdetachcloner listrrr,nnLinear requires_gradcopy_r)rrrWbnew_size new_layers rprune_linear_layerrs HHU\( ) )E !!#u--4466<<>>A z !88 !!##))++AA 5!((**0022AEL%%''((HJJHSM Xa[uz( R +,,&+I" 1<<>>***%)I" z', $Q\\^^,,,'+ $ r)r2r)r)=rrrGrrr r_rtypingrrrrrrr,torch.multiprocessingmultiprocessingr; packagingrrrKrarrFboolr&r1rCr>r?r@rsrvrnror}rjrrrrr lru_cacherrrrrModulerTensorrrrrrr LongTensorrr)rrr"s 222222222222 """"""%%%%%%@@@@@@    ? ? ?c ? ? ? ? ?77$7777 7C 7d 7 7 7 7$-$-c$-#$-$$-$-$-$-NsCx,000!!!   999<<<<        ???BGGG       @""""J UX_     ,&el*+,&,&,&  \ ,&,&,&,&^Cy#&36!#h+0S5<1G+H&#$eho#.(-r