
    6pj                         S SK r S SKrS SKJr  S SKrS SKJr  S SKJr  S SKJ	r	  S SK
Jr  \" 5       r " S S\5      r " S	 S
\5      rg)    N)islice)IterableDataset)tqdm)EXTENSIONS_TO_LOAD)
get_loggerc                   0    \ rS rSrSrS rS rS rS rSr	g)	ExternalDataset   z"Dataset class for custom datasets.c           	         Xl         [        R                  " U5      U l        U R                  R	                  SU R                   05        S U l        U R                   R                  5        VVs0 s H  u  p4U/ _M
     snnU l        0 U l        SnU R                   R                  5        GHU  u  pg[        U[        5      (       d  M  [        R                  R                  U5      (       d  MC  [        R                  " U5      n[        U V	s/ s H4  n	[        R                  R!                  U	5      S   R#                  S5      PM6     sn	5      n
SU
;   a  M  [%        U
5      S:w  aB  SR'                  [(        R*                  " 5       5      n[,        R/                  SU SU S	35        M  [1        U
5      S
   nU[(        ;  a  GM  U V	s/ s H"  n	[        R                  R'                  Xy5      PM$     nn	XR                  U'   GMX     U(       a?  [(        R2                  " U5      n[4        R6                  " U4SU R                  0UD6U l        g g s  snnf s  sn	f s  sn	f )Nsplit_config .   ,zSplit-z has been ignored, please flatten your folder structure, and make sure these files have same extensions. Supported extensions: z .r   
data_files)split_path_dictcopydeepcopyconfig_kwargsupdatespec_extension_datasetitemssplit_data_files
custom_map
isinstancestrospathisdirlistdirsetsplitextstriplenjoinr   keysloggererrorlistgetdatasetsload_dataset)selfr   r   k_file_ext
split_name	split_dirsplit_file_names	file_nameset_files_extssupported_extssplit_file_pathss                e/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/modelscope/msdatasets/dataset_cls/dataset.py__init__ExternalDataset.__init__   s(   .!]]=9!!>43G3G"HI&*# ,,224!
4 rE4!
  %)%9%9%?%?%A!J)S))bggmmI.F.F#%::i#8 !$%5&%5	 GG$$Y/399#>%5& " '~&!+%(XX.@.E.E.G%HNLL  -11?0@DE /2#55 &6$%5	 GGLL6%5 ! $ 5E%%j13 &B6 )--h7H*2*?*?+M%)%:%:+M>K+MD' G!
&$$s   -I	
;I
)Ic                     U R                   (       d  [        U R                  5      $ U R                   R                  5       $ N)r   r%   r   __len__r.   s    r9   r>   ExternalDataset.__len__B   s>     ,,   
 	
262M2M2U2U 3
	
    c                     U R                   (       d  U R                  R                  U5      $ U R                   R                  U5      $ r=   )r   r   r+   __getitem__)r.   items     r9   rC   ExternalDataset.__getitem__H   s9    **''++D11..::4@@rA   c              #      #    U R                   (       d)  U R                  R                  5        H
  u  pX4v   M     g U R                   R                  5        H
  u  pX4v   M     g 7fr=   )r   r   r   )r.   r/   vs      r9   __iter__ExternalDataset.__iter__N   sT     **,,224d
 5 3399;d
 <s   A$A&)r   r   r   r   r   N)
__name__
__module____qualname____firstlineno____doc__r:   r>   rC   rH   __static_attributes__ rA   r9   r	   r	      s    ,,M\
ArA   r	   c                   R   ^  \ rS rSrSrS
U 4S jjrS rS rS rS r	SS jr
S	rU =r$ )NativeIterableDatasetW   z&The modelscope iterable dataset class.c                 .   > [         TU ]  XUS9  X@l        g )N)ex_iterableinfosplit)superr:   stream_batch_size)r.   rU   rV   rW   rY   	__class__s        r9   r:   NativeIterableDataset.__init__Z   s    [5I!2rA   c              #      #    [        U R                  U R                  SS9SU R                  SS9 H  nU R	                  U5      nUv   M     g 7f)NF
batch_sizedrop_last_batchzOverall progressT)desctotaldynamic_ncols)r   iterrY   n_shards_download_item)r.   rD   rets      r9   rH   NativeIterableDataset.__iter__^   sU     		#55u  N'mm"$D %%d+CI$s   AAc                     U R                   $ r=   )rd   r?   s    r9   r>   NativeIterableDataset.__len__i   s    }}rA   c           	   #   :  #    [        U[        5      (       a
  UnUS-   nSnO$UR                  nUR                  nUR                  nUb  US::  a  [        S5      e[        [        U R                  SSS9UX45      SSS	9 H  nU R                  U5      nUv   M     g7f)
zP
Returns the item at index `index` in the dataset. Slice indexing is supported.
r   Nr   zstep must be positiveFr]   zSlicing progressT)r`   rb   )
r   intstartstopstep
ValueErrorr   r   rc   re   )r.   indexrl   rm   rn   rD   rf   s          r9   rC   !NativeIterableDataset.__getitem__l   s      eS!!E19DDKKE::D::D	455IIEIBE  ("$D %%d+CI$s   BBc                    0 n[        U[        5      (       a   UR                  5        H  u  p4XBU'   UR                  S5      (       d  M!  U R                  R
                  R                  S5      nUR                  U5      n[        U[        5      (       a  U/nXbU'   XBUR                  S5      '   M     U$ UnU$ ! [         a"  n[        R                  U5        Un S nAU$ S nAff = f)Nz:FILE
dl_manager)r   dictr   endswith_ex_iterablekwargsr+   download_and_extractr   r$   	Exceptionr(   r)   )r.   rD   rf   r/   rG   rs   ex_cache_pathes           r9   re   $NativeIterableDataset._download_item   s    dD!! JJLDAFzz'**%)%6%6%=%=%A%A,%O
(2(G(G(J%mS99-:OM!.A01AGGG,- )  
 C
  Q 
s   -B9 
A)B9 9
C%C  C%c                     / nUS::  a  U$ SnU R                  5        H!  nX1:  a    U$ UR                  U5        US-  nM#     U$ )z
Returns the first n rows of the dataset.

Args:
    n (int): Number of rows to return.

Returns:
    list: The list of results, e.g. [{'id': 'abc123', 'text': 'hello world'}, ...]
r   r   )rH   append)r.   nresiter_numrD   s        r9   headNativeIterableDataset.head   sW     6JMMOD} 
 JJtMH	 $
 
rA   )rY   )r   )   )rJ   rK   rL   rM   rN   r:   rH   r>   rC   re   r   rO   __classcell__)rZ   s   @r9   rR   rR   W   s)    03	4, rA   rR   )r   r   	itertoolsr   r,   r   	tqdm.autor   modelscope.utils.constantr   modelscope.utils.loggerr   r(   objectr	   rR   rP   rA   r9   <module>r      sC     	   $  8 .	Cf CLYO YrA   