
    6pj0                        S r SSKrSSKrSSKrSSKJr  SSKJrJrJ	r	  SSK
JrJr   SSKJrJrJrJrJrJrJrJr  SSKJr  Sr\(       d  SSKrSSKrSSKrSSKrSSKrSS
K J!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'  SSK(J)r)  S\*S\\	\*\*\*\\*   4      4S jr+S\*S\	\*\\*   4   4S jr,\RZ                  S4S\*S\*4S jjr.S\*4S jr/S\*S\*S\*S\*S\*4
S jr0S\*S\*S\*S\*S\\	\*\*4      S\\	\*\*4      S\*4S jr1S \*S\\	\*\*4      S\\	\*\*4      S\*S\*S\*S\*SS4S! jr2S\*S\*S\*S\*S\	\*\*4   4
S" jr3S#\*4S$ jr4\+r\.r\/r\4r\0r\2r\3rgg! \ a
    S	rSrSr GNf = f)%a  Compatibility shims for datasets>=4.0 script-based dataset loading.

Script-based dataset loading was removed in datasets 4.0. This module
provides minimal re-implementations of the necessary helpers so that
ModelScope can still load datasets that ship a custom builder .py script.

When running with datasets<4.0 the real implementations are simply
re-exported from datasets.load / datasets.utils.py_utils.
    N)Path)ListOptionalTuple)DownloadModeconfig)!HubDatasetModuleFactoryWithScript#LocalDatasetModuleFactoryWithScriptresolve_trust_remote_code_get_importable_file_path_create_importable_file_load_importable_fileinit_dynamic_modulesfiles_to_hash)get_importsTF)urlparse)_hash_python_lines)url_or_path_join)hf_dataset_url)FileLock	file_pathreturnc                 P  ^	 [        U SS9 nUR                  5       nSSS5        / nSnW GH  n[        R                  " SU5      n[	        U5      S:X  a  U(       + nU(       a  M:  [        R
                  " SU[        R                  S9m	T	c$  [        R
                  " S	U[        R                  S9m	T	c  M  T	R                  S5      (       a  [        U	4S
 jU 5       5      (       a  M  T	R                  S5      (       aD  T	R                  S5      n[        U5      u  pxUR                  ST	R                  S5      Xx45        GM  T	R                  S5      (       a6  UR                  ST	R                  S5      T	R                  S5      S45        GMb  GMe  T	R                  S5      (       a6  UR                  ST	R                  S5      T	R                  S5      S45        GM  UR                  ST	R                  S5      T	R                  S5      S45        GM     U$ ! , (       d  f       GN = f)zHParse a dataset script for import statements (ported from datasets<4.0).utf-8encodingNFz[\s\S]*?"""[\s\S]*?   z=^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*))flagszQ^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)c              3   P   >#    U  H  oS    TR                  S5      :H  v   M     g7f)r      N)group).0impmatchs     _/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/modelscope/msdatasets/utils/_compat.py	<genexpr>&_compat_get_imports.<locals>.<genexpr>U   s      C7C1vQ/7s   #&   externalr    internallibrary)open	readlinesrefindalllenr$   	MULTILINEr!   any_compat_convert_github_urlappend)
r   flinesimportsis_in_docstringlinedocstr_start_matchurl_pathsub_directoryr$   s
            @r%   _compat_get_importsr=   ;   s    )g.!KKME / >@D!#,BD!I%&!+&5"5HHPll$E }h,,( }{{1~~C7CCC;;q>>${{1~H.H /"+HNN#U[[^XMO[[^^NN#U[[^U[[^TJL $ ;;q>>NN"EKKNEKKNDIK NN"EKKNEKKNDIKE H S /.s   H
H%r;   c                    [        U 5      nS nUR                  S;   a  UR                  S:X  a  SU ;   a:  U R                  S5      (       d  [	        SU  S35      eU R                  SS5      n X4$ UR                  SS  nS	U;   a  UR                  S	5      OUS
4u  pEUR                  S5      u  pgSU SU SU S3n U SU 3nX4$ )N)httphttpss3z
github.comblob.pyzExternal import from github at z should point to a .py filerawr   z/tree/master/zhttps://github.com/z	/archive/z.zip-)r   schemenetlocendswith
ValueErrorreplacepathsplit)r;   parsedr<   github_path	repo_infobranch
repo_owner	repo_names           r%   r3   r3   i   s    (#== # #'-}}'D!((//$9(C^_  $++FE: && %kk!"o3;{3JK%%h/ (+ "	 )2(<%
0Ai[	RXQYY]^#,+Qvh 7&&    namec                    [        U=(       d    [        R                  5      nU[        R                  ;  a  [        R                  R                  U5        [        R                  " USS9  [        R                  R                  US5      n[        R                  R                  U5      (       d+  [        US5          S S S 5        [        R                  " 5         [        R                  R                  X5      n[        R                  " USS9  [        R                  R                  US5      n[        R                  R                  U5      (       d  [        US5          S S S 5        U$ U$ ! , (       d  f       N= f! , (       d  f       U$ = f)NTexist_ok__init__.pyw)strr   HF_MODULES_CACHEsysrM   r4   osmakedirsjoinexistsr,   	importlibinvalidate_caches)rV   hf_modules_cache	init_pathdynamic_modules_path
init_path2s        r%   _compat_init_dynamic_modulesri      s    /J63J3JK388+HHOO,-
$t4GGLL!1=A	ww~~i((i% &'')!ww||,<C
(48WW\\"6F
ww~~j))j#& '#### &% '&##s   ;E,E=,
E:=
Fc                    / nU  Hl  n[         R                  R                  U5      (       a4  UR                  [	        [        U5      R                  S5      5      5        M[  UR                  U5        Mn     / nU H5  n[        USS9 nUR                  UR                  5       5        S S S 5        M7     [        U5      $ ! , (       d  f       MS  = f)Nz
*.[pP][yY]r   r   )r_   rM   isdirextendlistr   rglobr4   r,   r-   r   )
file_pathsto_use_filesfpr6   r5   s        r%   _compat_files_to_hashrr      s    Bww}}R  ##Db)E$FG##B'	 
 Bb7+qQ[[]+ ,+  "%(( ,+s    B==
C	rg   module_namespacesubdirectory_namec                     [         R                  R                  XUR                  SS5      5      n[         R                  R                  XBUR	                  S5      S   S-   5      $ )NrF   --rC   )r_   rM   ra   rL   rN   )rg   rs   rt   rV   importable_dirs        r%    _compat_get_importable_file_pathry      sS     &:&*ll3&=?ww||N JJsOB/%79 	9rU   importable_directory_pathoriginal_local_pathlocal_importsadditional_filesc                 X   [         R                  R                  UU5      n[         R                  R                  XpS-   5      nUS-   n	[        U	5         U[        R
                  :X  a:  [         R                  R                  U5      (       a  [        R                  " U5        [         R                  " USS9  [         R                  R                  US5      n
[         R                  R                  U
5      (       d  [        U
S5          S S S 5        [         R                  " USS9  [         R                  R                  US5      n[         R                  R                  U5      (       d  [        US5          S S S 5        [         R                  R                  U5      (       d  [        R                  " X85        [         R                  R                  U5      S   S-   n[         R                  R                  U5      (       d/  UUS	.n[        USS
S9 n[        R                  " X5        S S S 5        U GH  u  nn[         R                  R                  U5      (       ab  [         R                  R                  UUS-   5      n[         R                  R                  U5      (       d  [        R                  " UU5        M  M  [         R                  R!                  U5      (       a`  [         R                  R                  X5      n[         R                  R                  U5      (       d  [        R"                  " UU5        GM  GM  [%        SU 35      e   U H  u  nn[         R                  R                  UU5      n[         R                  R                  U5      (       a  [&        R(                  " UU5      (       a  Mh  [        R                  " UU5        M     S S S 5        U$ ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       U$ = f)NrC   z.lockTrX   rZ   r[   r   z.json)zoriginal file pathzlocal file pathr   r   zError with local import at )r_   rM   ra   r   r   FORCE_REDOWNLOADrb   shutilrmtreer`   r,   copyfilesplitext_jsondumpisfilerk   copytreeImportErrorfilecmpcmp)rV   rz   rt   r{   r|   r}   download_modeimportable_subdirectoryimportable_file	lock_pathinit_fpinit_fp2	meta_pathmetamfimp_nameimp_pathdest	file_nameoriginal_path	dest_paths                        r%   !_compat_copy_script_and_resourcesr      s    #%'',,/H/@#B'',,'>uM-7	i  = =="''..-C/ C/78KK1DAggll#<mLG77>>'**'3' (KK/$?ww||$;]KH77>>(++(C( )77>>/22 3EGG,,_=a@7J	ww~~i00.A+:D iw?2

4, @&3"(77>>(++77<<(?(05(8:D77>>$//$7 0WW]]8,,77<<(?JD77>>$//$7 0 &(CH:&NOO '4 -=(	=GGLL)@)L	ww~~i00%y92 92OOM9=	 -=I !R E ('
 )( @?- ! R s^   B4PO$	A-P6O68BPP*FP?P$
O3	.P6
P	 P
P	P
P)
local_pathc           
      "   [         R                  R                  X4UR                  SS5      5      n[	        U5      R                  SSS9  [	        U5      R                  S-  R                  SS9  [        UR                  S5      S   UUU UUUS9  g )	NrF   rv   T)parentsrY   rZ   rX   rw   )rV   rz   rt   r{   r|   r}   r   )
r_   rM   ra   rL   r   mkdirparenttouchr   rN   )	r   r|   r}   rg   rs   rt   rV   r   rx   s	            r%   _compat_create_importable_filer      s     &:&*ll3&=?^""4$"?	n		$	$}	4;;T;J)C$&4/ *'-'	
rU   c           	          SR                  [        R                  R                  U 5      UUR	                  SS5      UUR                  S5      S   /5      nXB4$ )N.rF   rv   rw   )ra   r_   rM   basenamerL   rN   )rg   rs   rt   rV   module_paths        r%   _compat_load_importable_filer      sY     hhGG12LLd#JJsOB 
  --rU   repo_idc                 *    U c  [        SU S35      eU $ )NzThe repository for z contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at the Hub.
Please pass the argument `trust_remote_code=True` to allow custom code to be run.)rK   )trust_remote_coder   s     r%   !_compat_resolve_trust_remote_coder     s.    $%gY /2 34 4
 ! rU   )5__doc__rc   r_   r^   pathlibr   typingr   r   r   datasetsr   r   datasets.loadr	   r
   r   r   r   r   r   r   datasets.utils.py_utilsr   _HAS_SCRIPT_LOADINGr   r   hashlibjsonr   r.   r   urllib.parser   datasets.packaged_modulesr   datasets.utils.file_utilsr   datasets.utils.hubr   filelockr   r\   r=   r3   MODULE_NAME_FOR_DYNAMIC_MODULESri   rr   ry   r   r   r   r    rU   r%   <module>r      s|    	 
  ( ( )
/	 	 	 4 %<:1!,,#E#sC#*F$GH,\'S 'U3;M5N '0 ::$$ 
$*)S )	9!	9	9 	9 		9
 
	966#&6 6 !	6
 E#s(O,6 uS#X/6 
6p

E#s(O,
 uS#X/
 "	

 
 
 
 

0.!.. . 	.
 
sCx."!c ! &K7)M A @<8e   /(,%*.'/s   E EE