
    RЦi                      $   S r SSKrSSKrSSKJrJr  SSKJr  SSKrSSK	J
r  SSKJr   SSKrSSKJr  SSKJr  S	S
KJr  S	SKJr  S	SKJr  \" \R:                  R=                  SS5      5      r " S S\5      r g! \ a  r\" S5        \eSrCff = f)z&Dataset reader for HF IterableDataset
    N)repeatchain)Optional)Image)split_dataset_by_node)	SplitInfozDPlease install Hugging Face datasets package `pip install datasets`.   )load_class_map)Reader)SharedCountHFIDS_SHUFFLE_SIZEi   c            !          ^  \ rS rSr               SS\S\\   S\S\S\S\S\S	\S
\\   S\S\S\S\S\\   S\\   S\4 U 4S jjjr	S r
 SS\\   4S jjrS rS rS rS rSS jrSS jrSrU =r$ ) ReaderHfids   namerootsplitis_training
batch_sizedownloadrepeatsseed	class_map	input_keyinput_img_mode
target_keytarget_img_modeshuffle_sizenum_samplestrust_remote_codec                   > [         TU ]  5         X l        X0l        X@l        XPl        X`l        Xpl        Xl        U=(       d    [        U l
        Xl        Xl        Xl        Xl        [        R                   " UUUS9U l        U(       a  U R"                  R%                  5         S nU R"                  R&                  R(                  (       a|  X0R"                  R&                  R(                  ;   aY  [+        U R"                  R&                  R(                  U   [,        5      (       a#  U R"                  R&                  R(                  U   nU(       a  Xl        O5U(       a#  UR0                  (       a  UR0                  U l        O[3        S5      eSU l        U	(       a  [7        U	5      U l        SU l        O0 U l        SU l        SU l        [>        R@                  " 5       (       ag  [>        RB                  " 5       (       aM  [>        RD                  " 5       S:  a4  [>        RF                  " 5       U l        [>        RD                  " 5       U l        S U l$        SU l%        SU l&        SU l'        SU l(        S U l)        [U        5       U l+        g )N)	cache_dirr    zDataset length is unknown, please pass `num_samples` explicitly. The number of steps needs to be known in advance for the learning rate scheduler.FTr   r	   ),super__init__r   r   r   r   r   r   common_seedSHUFFLE_SIZEr   r   r   r   r   datasetsload_dataset_builderbuilderdownload_and_prepareinfosplits
isinstancer   r   num_examples
ValueErrorremap_classr
   class_to_idx	dist_rankdist_num_replicasdistis_availableis_initializedget_world_sizeget_rankworker_info	worker_idnum_workersglobal_worker_idglobal_num_workersdsr   epoch)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    
split_info	__class__s                     ]/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/data/readers/reader_hfids.pyr$   ReaderHfids.__init__   s   & 		
&$ (8L",$.44/

 LL--/*.
<<##1B1B1I1I(I$,,++22599EE26,,2C2C2J2J52Q
*J33)66Dd 
 ! .y 9D#D "D !"4#6#6#8#8T=P=P=RUV=V!]]_DN%)%8%8%:D"   !"# 7; ]
    c                 $    XR                   l        g N)r?   value)r@   counts     rC   	set_epochReaderHfids.set_epochp   s     

rE   r;   c                 p    U R                   b  g Ub%  Xl        U R                  U R                  -  U l        g g rG   )r>   r;   r3   r=   )r@   r;   s     rC   set_loader_cfgReaderHfids.set_loader_cfgt   s;     77"*&*&<&<t?O?O&OD# #rE   c                    U R                   c  [        R                  R                  R	                  5       nUb(  Xl         UR
                  U l        UR                  U l        U R                  U R                  -  U l	        U R                  U R                  -  U R                  -   U l        U R                  (       a=  U R                  R                  U R                  S9nUR!                  U R                  S9nO#U R                  R#                  U R                  S9nU R$                  (       a$  UR'                  U R(                  U R*                  S9n[-        X0R                  U R                  S9U l        g)z7Lazily initialize worker (in worker processes)
        N)r   )
num_shards)r   buffer_size)rank
world_size)r9   torchutilsdataget_worker_infoidr:   r;   r3   r=   r2   r<   r   r)   
as_datasetr   to_iterable_datasetas_streaming_datasetr   shuffler%   r   r   r>   )r@   r9   datasetr>   s       rC   
_lazy_initReaderHfids._lazy_init~   s    #++**::<K&#. !,#.#:#: &*&<&<t?O?O&OD#$(NNT5E5E$E$VD!==ll--DJJ-?G,,8O8O,PB 222DB!1!1t?P?PQB (DLbLbcrE   c                    [        SU R                  5      U R                  -  [        U R                  U R                  5      -  nU R
                  (       d  U R                  S:  a  [        R                  " U5      nU R
                  (       a<  U R                  b/  [        R                  " XR                  -  5      U R                  -  n[        U5      $ )Nr	   )
maxr   r   r=   r3   r   mathceilr   int)r@   num_worker_sampless     rC   _num_samples_per_worker#ReaderHfids._num_samples_per_worker   s    4<< 4#3#33c$:Q:QSWSiSi6jj 	t559!%+=!> ;!%+=+O!PSWSbSb!b%&&rE   c              #     #    U R                   c  U R                  5         U R                   R                  U R                  R                  5        U R                  5       nSnU R                  (       a*  [        R                  " [        U R                   5      5      nO[        U R                   5      nU GH  nX@R                     nU R                  (       a5  UR                  U R                  :w  a  UR                  U R                  5      nX@R                     nU R                   (       a\  [#        U[$        R$                  5      (       d   S5       eUR                  U R                   :w  a  UR                  U R                   5      nO U R&                  (       a  U R(                  U   nXV4v   US-  nU R                  (       d  GM  X!:  d  GM    g    g 7f)Nr   z7target_img_mode is specified but target is not an imager	   )r>   r^   rJ   r?   rH   rf   r   r   from_iterabler   iterr   r   modeconvertr   r   r-   r   r0   r1   )r@   target_sample_countsample_countds_itersample
input_datatarget_datas          rC   __iter__ReaderHfids.__iter__   sY    77?OO$****+"::<))&/:G477mGF&,^^&<J""z$:M:M'M'//0C0CD
 1K##!+u{{;;v=vv;##t';';;"-"5"5d6J6J"KK!!"//<))ALL$G s   F=GGGc                 @    U R                  5       U R                  -  nU$ rG   )rf   r;   )r@   r   s     rC   __len__ReaderHfids.__len__   s!    224t7G7GGrE   c                      S5       e)NzNot supported )r@   indexbasenameabsolutes       rC   	_filenameReaderHfids._filename   s    %o%urE   c                     U R                   c  U R                  5         / nU R                    HK  nSU;   a  US   nO+SU;   a  US   nOSU;   a  US   nOSU;   a  US   nO S5       eUR                  U5        MM     U$ )z/Return all filenames in dataset, overrides base	file_namefilenamerX   image_idzNo supported name field present)r>   r^   append)r@   r{   r|   namesrp   r   s         rC   	filenamesReaderHfids.filenames   s    77?OOggFf$k*v%j)d|v%j)???uLL  rE   )r   r)   r1   r%   r3   r2   r   r>   r?   r=   r<   r   r   r   r   r;   r0   r   r   r   r   r   r   r:   r9   )NtrainFr	   Fr   *   NimageRGBlabel NNFrG   )FF)__name__
__module____qualname____firstlineno__strr   boolrd   dictr$   rJ   rM   r^   rf   rs   rv   r}   r   __static_attributes____classcell__)rB   s   @rC   r   r      s?    #'  %"(,$"'%#%*.)-&+#P#P# 3-P# 	P#
 P# P# P# P# P#  ~P# P#  P# P# !P# #3-P#  "#!P#"  $#P# P#d! *.P!#PdB'8& rE   r   )!__doc__rb   os	itertoolsr   r   typingr   rT   torch.distributeddistributedr4   PILr   r'   datasets.distributedr   datasets.splitsr   ImportErroreprintr   r
   readerr   shared_countr   rd   environgetr&   r   ry   rE   rC   <module>r      s     	 #     :) &  % 2::>>"6=>&   	
PQ
Gs   A: :B 
B

B