
    Ƒi                     R    S SK r S SKJr  S SKJr  S SKJr  S SKJ	r	   " S S\
5      rg)    N)Pool)tqdm)OssUtilities)
UploadModec                   j    \ rS rSrS\S\S\4S jrS\S\S\S	\4S
 jrS\S\S\S\S\	S\S	\4S jr
Srg)DatasetUploadManager   dataset_name	namespaceversionc                 .    SSK Jn  [        XUS9U l        g )Nr   )HubApi)r
   r   revision)modelscope.hub.apir   r   oss_utilities)selfr
   r   r   r   s        h/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/modelscope/msdatasets/utils/upload_utils.py__init__DatasetUploadManager.__init__   s    -)%WN    object_namelocal_file_pathupload_modereturnc                 >    U R                   R                  UUSUS9nU$ )NToss_object_namer   indicate_individual_progressr   r   upload)r   r   r   r   
object_keys        r   r    DatasetUploadManager.upload   s1    ''..'+)-#	 / %

 r   object_dir_namelocal_dir_pathnum_processes	chunksizefilter_hidden_filesc                 <  ^ ^ U U4S jn/ n[         R                  " U5       H  u  pnU H  nU(       a  UR                  S5      (       a  M"  [         R                  R	                  UU	R                  USS5      R                  S5      U5      n[         R                  R	                  X5      nUR                  X45        M     M     [        US9 n[        [        UR                  XxUS9[        U5      S95      nS S S 5        [        W5      $ ! , (       d  f       N= f)	Nc                 J   > TR                   R                  U S   U S   STS9  g )Nr      Fr   r   )argsr   r   s    r   
run_upload3DatasetUploadManager.upload_dir.<locals>.run_upload!   s0    %% $Q $Q-2'	 & )r   . r*   /)	processes)r&   )total)oswalk
startswithpathjoinreplacestripappend
ThreadPoollistr   imaplen)r   r#   r$   r%   r&   r'   r   r,   
files_listrootdirsfiles	file_namer   r   poolresults   `     `          r   
upload_dirDatasetUploadManager.upload_dir   s    	) 
!#!8D"	&9+?+?+D+D ggll#LLQ7==cBIO #%'',,t"?!!;"@A # "9 -0DIIj	IJj/+,F 1 6{ 10s   +D
D)r   N)__name__
__module____qualname____firstlineno__strr   r   r    intboolrF   __static_attributes__ r   r   r   r      s~    NS NS N3 N#  &+.# s "%25(,;EJMr   r   )r3   multiprocessing.dummyr   r;   	tqdm.autor   %modelscope.msdatasets.utils.oss_utilsr   modelscope.utils.constantr   objectr   rP   r   r   <module>rV      s"    
 4  > 0/6 /r   