
    6pjd8                        S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SK	J
r
  S SKJr  S SKJrJr  S SKJrJr  S SKrS SKJr  S SKJr  S S	KJrJrJrJr  S S
KJr  S SKJ r   S SK!J"r"  S SK#J$r$  S SK%J&r&  \"" 5       r'SS\\\(\)4      S\(4S jjr*    SS\(S\(S\+S\,S\,S\,S\RZ                  4S jjr. SS\RZ                  4S jjr/ SS\\RZ                     4S jjr0               S S\(4S jjr1g)!    N)contextmanager)partial)Path)OptionalUnion)urljoinurlparse)tqdm)config)hash_url_to_filename"get_authentication_headers_for_urlfsspec_head
fsspec_get)FileLock)MS_DATASETS_CACHE)
get_logger)HubApi)__version__
user_agentreturnc                 2   S[          3nUS[        R                   3-  nUS[        R                   3-  n[        R                  (       a  US[        R
                   3-  n[        R                  (       a  US[        R                   3-  n[        R                  (       a  US[        R                   3-  n[        U [        5      (       a.  USSR                  S U R                  5        5       5       3-  nU$ [        U [        5      (       a  USU -   -  nU$ )	Nz	datasets/z	; python/z
; pyarrow/z; torch/z; tensorflow/z; jax/z; c              3   4   #    U  H  u  pU S U 3v   M     g7f)/N ).0kvs      e/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/modelscope/msdatasets/utils/hf_file_utils.py	<genexpr>-get_datasets_user_agent_ms.<locals>.<genexpr>-   s     G4FDAs!A3Z4Fs   )r   r   
PY_VERSIONPYARROW_VERSIONTORCH_AVAILABLETORCH_VERSIONTF_AVAILABLE
TF_VERSIONJAX_AVAILABLEJAX_VERSION
isinstancedictjoinitemsstr)r   uas     r   get_datasets_user_agent_msr/   "   s   [M	"BIf''(
))BJv--.
//B
--.//
f//011
v))*++*d##
499GJ4D4D4FGGHII I 
J	$	$
dZI    methodurlmax_retriesbase_wait_timemax_wait_timetimeoutc           	         Su  pxSn	UR                  S5      =(       d    0 R                  SS5      n
U(       d  US-  n [        R                  SXXZ=(       d    S5        [        R                  " 5       n[
        R                  " SU R                  5       XS	.UD6n	[        R                  " 5       U-
  n[        R                  S
U	R                  U	R                  R                  SS5      UU5        SnU(       d  M  U	$ ! [
        R                  R                  [
        R                  R                  [
        R                  R                  4 a  n[        R                  S[        U5      R                   XX]5        Xr:  a  Ue[        R#                  U  SU SXr-   S35        [%        XCSUS-
  -  -  5      n[        R&                  " U5         SnANSnAff = f)a  Wrapper around requests to retry in case it fails with a ConnectTimeout,
ReadTimeout or ConnectionError, with exponential backoff.

Note that if the environment variable HF_DATASETS_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised.

Args:
    method (str): HTTP method, such as 'GET' or 'HEAD'.
    url (str): The URL of the resource to fetch.
    max_retries (int): Maximum number of retries, defaults to 0 (no retries).
    base_wait_time (float): Duration (in seconds) to wait before retrying the first time. Wait time between
        retries then grows exponentially, capped by max_wait_time.
    max_wait_time (float): Maximum amount of time between two retries, in seconds.
    **params (additional keyword arguments): Params to pass to :obj:`requests.request`.
)r   FNheadersRange    z][MS_DOWNLOAD] _request_with_retry_ms sending request: method=%s, url=%s, timeout=%s, Range=%szN/A)r1   r2   r6   zb[MS_DOWNLOAD] _request_with_retry_ms response: status=%s, content_length=%s, elapsed=%.3fs, url=%sContent-LengthTzP[MS_DOWNLOAD] _request_with_retry_ms %s: method=%s, url=%s, timeout=%s, error=%sz request to z timed out, retrying... []   r   )getloggerdebugtimeperf_counterrequestsrequestupperstatus_coder8   
exceptionsReadTimeoutConnectTimeoutConnectionErrorerrortype__name__infominsleep)r1   r2   r3   r4   r5   r6   paramstriessuccessresponserange_headert0elapsederr
sleep_times                  r   _request_with_retry_msr[   3   s   . NEHJJy)/R44WbAL
	'LL:W&;e
 ""$B''bv||~3b[abH'')B.GLLF$$  $$%5u= G' gD O ##//##22##335 	' LL:S	""F
 "	vhl3%7PQVQbPccdef uqyAQ0QR


:&&	's   B3C< <AGA<GGc                     [         R                  " U5      =(       d    0 n[        UR                  S5      S9US'   [	        SU UUUUUUS9nU$ )N
user-agentr   HEAD)r1   r2   proxiesr8   cookiesallow_redirectsr6   r3   )copydeepcopyr/   r?   r[   )r2   r`   r8   ra   rb   r6   r3   rU   s           r   http_head_msre   r   sX     mmG$*G6'++lB[\GL%'	H Or0   c	                    [         R                  SXU5        Ub  [        U5      O0 n[        UR	                  S5      S9US'   US:  a
  SUS S3US'   [        S	U S
UUUUUS9n	Uc  U	$ U	R                  S:X  a  g U	R                  R	                  S5      n
U
b  U[        U
5      -   OS n[        XS
SU=(       d    SS9nU	R                  SS9 H.  nUR                  [        U5      5        UR                  U5        M0     UR                  5         g )NzC[MS_DOWNLOAD] http_get_ms entry: url=%s, timeout=%s, resume_size=%sr]   r^   r   zbytes=d-r9   GETT)r1   r2   streamr`   r8   ra   r3   r6   i  r<   BDownloading)totalinitial
unit_scaleunitdesci   )
chunk_size)r@   rA   r*   r/   r?   r[   rG   r8   intr
   iter_contentupdatelenwriteclose)r2   	temp_filer`   resume_sizer8   ra   r6   r3   rq   rU   content_lengthrm   progresschunks                 r   http_get_msr~      s#    LLMk  '2d7mG6'++lB[\GLQ#K?!4%	H s"%%))*:;N1?1KK#n--QUE%CVZVk^klH&&$&7E
# 8 NNr0   c                   ^% US:w  a!  [         R                  " SU S3[        5        Un
Uc  [        n[	        U[
        5      (       a  [        U5      n[        R                  " USS9  U(       a   [        U [        U 5      R                  5      nOU nSnSnSnSnSnSn[        USS9n[        R                  R                  UU5      nUc  S	U-   S
-   n[        R                  R                  U5      (       a  U(       d	  U(       d  U$ [        X
S9nUb  UUS'   U(       Gd  [        U 5      R                   nUS;  a?  [#        XS9nU(       a+  UR%                  SS5      =(       d    UR%                  SS5      OSnSn ['        5       R)                  U
S9n[+        U SUUU	UUS9nUR,                  S:X  a  U(       a  UR.                  R%                  S5      OSnUR0                  R3                  5        H:  u  nnUR5                  S5      (       d  M  SU ;   d  M&  U SU-   -  n UR0                  nM<     SnSU ;   a  SU ;  a  U S-  n OUR,                  S:X  a  SU ;   d~  UR,                  S:X  a  SU ;   dh  UR,                  S:X  aB  [6        R8                  " SU 5      (       d<  [6        R8                  " SUR:                  5      (       d  UR,                  S:X  a!  SU ;   a  Sn[<        R?                  S U  35        O6UR,                  S!:X  a&  [@        RB                  U ;   a  U
c  [E        S"U  S#35      eU(       d  [        R                  R                  U5      (       a	  U(       d  U$ U(       a  [O        S$U S%35      eUb  UR,                  S&:X  a  [O        S'U  35      eUb  [E        S(U  S)[Q        U5       S*35      eUb  [E        S(U  S+UR,                   S*35      e[E        S(U  35      e[        UU5      n[        R                  R                  UU5      n[        R                  R                  U5      (       a	  U(       d  U$ US,-   n[S        U5         [        R                  R                  U5      (       a  U(       d  UsSSS5        $ US--   m%[T        S>U%4S. jj5       nS/nU(       aN  [W        US0S19n[        R                  R                  T%5      (       a   [        RX                  " T%5      RZ                  nU" 5        n US;  a  []        U U XS29  O&[<        R?                  S3U 5        [_        U U UUUUU	US49  SSS5        [<        R?                  S5U  S6U 35        [`        Rb                  " W Rd                  U5        [        Rf                  " S75      n![        Rf                  " U!5        [        Rh                  " US7U!) -  5        [<        R?                  S8U 35        U US9.n"US:-   n#[k        U#S;S<S=9 n$[l        Rn                  " U"U$5        SSS5        SSS5        U$ ! [F        [H        RJ                  RL                  4 a  nUn SnAGNSnAff = f! , (       d  f       GN= f! , (       d  f       N^= f! , (       d  f       U$ = f)?ad  
Given a URL, look for the corresponding file in the local cache.
If it's not there, download it. Then return the path to the cached file.

Return:
    Local path (string)

Raises:
    FileNotFoundError: in case of non-recoverable file
        (non-existent or no cache on disk)
    ConnectionError: in case of unreachable url
        and no cache on disk

deprecatedz'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.
You can remove this warning by passing 'token=z
' instead.NT)exist_okF)etagzDownloading [r=   )tokenr]   )httphttps)storage_optionsETagr   )access_token)rb   r`   r6   r3   r8   ra      download_warningzdrive.google.comz	&confirm=zconfirm=z
&confirm=ti  zfirebasestorage.googleapis.comi  i  z7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url i  zUnauthorized for URL zU. Please use the parameter `token=True` after logging in with `huggingface-cli login`z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.i  zCouldn't find file at zCouldn't reach z ()z (error z.lockz.incompletec              3   b   >#    [        TU 5       nUv   S S S 5        g ! , (       d  f       g = f7fN)open)modefincomplete_paths     r   temp_file_manager,get_from_cache_ms.<locals>.temp_file_managerL  s"     ot, -,,s   /	/
,/r   za+b)r   )r   rq   z3[MS_DOWNLOAD] get_from_cache_ms downloading: url=%s)ry   r`   rz   r8   ra   r3   rq   zstoring z in cache at i  zcreating metadata file for )r2   r   z.jsonwzutf-8)encoding)zw+b)8warningswarnFutureWarningr   r)   r   r-   osmakedirsr   r	   pathr   r+   existsr   schemer   r?   r   get_cookiesre   rG   r8   ra   r,   
startswithrematchr2   r@   rO   r   HF_ENDPOINTrK   OSErrorrD   rH   TimeoutFileNotFoundErrorreprr   r   r   statst_sizer   r~   shutilmovenameumaskchmodr   jsondump)&r2   	cache_dirforce_downloadr`   etag_timeoutresume_downloadr   local_files_onlyuse_etagr3   r   use_auth_tokenignore_url_paramsr   download_descdisable_tqdm
cached_url	connectedrU   ra   r   
head_errorr   filename
cache_pathr8   r   r   e	lock_pathr   rz   ry   r   meta	meta_path	meta_filer   s&                                        @r   get_from_cache_msr      s    > %==K<LJX	

 %	)T""	N	KK	D)S(3-"4"45

IHGDJF $JT:Hi2J'(2S8	ww~~j!!. 1BG * #%%**"3HHQYHLL.L(,,vt2L_cDI-	h***>G# $$'H ##s*7?x''++F3T$,,224DAq||$677<NRU<U{Q."*"2"2	 5
 !	%,31F<'C %%,1QUX1X((C/4F#4M((C/![]`aa88$JHLLYY ((C/4OSV4V 	@FG%%,1C1Cs1Ju}%+C5 1j k  77>>*%%n#H U^ ^  !h&:&:c&A#&<SE$BCC!!OC54
;K:LA"NOO!!OC5AUAU@VVW"XYY!OC5"9:: $J5Hi2J	ww~~j!!. W$I	)	77>>*%%n 
	
 %}4		 
	  '(9 Fww~~o.. ggo6>>  I ..3	?_QSVW'# +## +&	 !$ 	hse==>INNJ/

UeV^,1*>?D)(	)S73yIIdI& 4_ 
d i ,,445 	J	l ! 6 43_ 
	d sv   9BX	 X	 +X	 ?B.X	 .6X	 -Y :A4Y .9X='B8Y Y7Y 	$X:-X55X:=
Y	Y 
Y	Y  
Y/r   )r>   g      ?         $@)NNNTr   r   )Nr   NNg     r@r   N)NFNd   FNFTr   Nr   FNNN)2r   r   r   rc   r   rB   r   
contextlibr   	functoolsr   pathlibr   typingr   r   urllib.parser   r	   rD   	tqdm.autor
   datasetsr   datasets.utils.file_utilsr   r   r   r   filelockr   modelscope.utils.config_dsr   modelscope.utils.loggerr   modelscope.hub.apir   
modelscoper   r@   r-   r*   r/   rs   floatResponser[   re   r~   r   r   r0   r   <module>r      sN  
  	 	     %   " *   @ @  8 . % "	8E#t)4D+E QT ( <<	< < 	<
 < < <@ de& qu!h !L 
!O" 	#Or0   