
    RЦiPB                        S r SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKJ	r	  SSK
Jr  SSKJr  SSKJrJrJrJrJrJr  SSKrSSKJr  SSKrSSKJr  SSKJrJrJr   SSK r!SS	K"J#r#J$r$  SS
K%J&r&  SSK'J(r(J)r)J*r*J+r+  SSK-J.r.  SSK/J0r0  SSK1J2r2  \Rf                  " \45      r5\6" \Rn                  Rq                  SS5      5      r9\6" \Rn                  Rq                  SS5      5      r:S#S jr;\	 " S S5      5       r<S\=S\4S jr>S r?    S$S jr@S rA\!b   " S S\!R                  5      rCOSrC " S S \5      rD " S! S"\05      rEg! \, a    Sr!Sr& Nf = f)%zQDataset reader for webdataset

Hacked together by / Copyright 2022 Ross Wightman
    N)	dataclass)partial)islice)AnyCallableDictListOptionalTuple)Image)DatasetIterableDatasetget_worker_info)_shufflegetfirst)expand_urls)base_plus_ext
url_openertar_file_expandervalid_sample   )load_class_map)Reader)SharedCountWDS_SHUFFLE_SIZEi    WDS_INITIAL_SIZEi   c                    [        U[        5      (       a  U4n/ nSnU H  n[        R                  R	                  X5      n UR                  U5        [        R                  " U5       nUR                  S5      (       a  [        R                  " U5      nO[        R                  " U5      nS S S 5        Ws  $    [        R                  SU SU S35        0 $ ! , (       d  f       N2= f! [         a  n[        U5      n S nAM  S nAff = f)N z.jsonzDataset info file not found at z	. Error: z.. Falling back to provided split and size arg.)
isinstancestrospathjoinappendwdsgopenendswithjsonloadyaml	safe_load	Exception_loggerwarning)	rootnamestriederr_strn	full_pathf	info_dictes	            [/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/data/readers/reader_wds.py
_load_infor9   *   s    %EGGGLL)			LL#9%::g&& $		!I $q 1I	 &
   OO
)%	' C7 	78 I &%  	!fG	s1   'C*)AC-	C*
C'	#C**
D
4DD
c                   `    \ rS rSr% \\S'   \\   \S'   Sr\\   \S'   Sr	\\S'   Sr
\\S'   Srg	)
	SplitInfoB   num_samples	filenames shard_lengthsr   	alt_labelnameN)__name__
__module____qualname____firstlineno__int__annotations__r   r    r@   rA   rB   __static_attributes__r?       r8   r;   r;   B   s3    Sz "M5:"IsD#NrJ   r;   splitinfoc           	      (  ^ S nSU ;   d  SU ;   Ga*  U R                  S5      n SnSn[        U 5      S:  a  [        U S   5      nU S   n SU ;  a<  U R                  S	S
5      n[        U5      S-
  nU(       a  SU;   a  XV   US   ;   a  XV   n[        U 5      nU(       a  US   U   nU(       dx  [	        US   US   5       V	V
s0 s H  u  pX_M	     sn
n	m[        U4S jU 5       5      n[        TR                  5       5      US'   [        TR                  5       5      US'   X8S'   U" U5      nU$ [        UUUS9n U$ SU;  d  XS   ;  a0  [        SU  SUR                  S0 5      R                  5        S35      eU n US   U    nU" U5      nU$ s  sn
n	f )Nc           	      x    [        U S   [        U S   5      [        U S   5      U R                  SS5      U S   S9$ )Nr=   r>   r@   rA   r   rB   )r=   r>   r@   rA   rB   )r;   tupleget)	dict_infos    r8   _info_convert(_parse_split_info.<locals>._info_convertL   sG    !-0Ik23	/ :;mmK46"
 	
rJ   tarz..|r   r   r   z::-   splitsr>   r@   c              3   .   >#    U  H
  nTU   v   M     g 7fNr?   ).0r5   _fcs     r8   	<genexpr>$_parse_split_info.<locals>.<genexpr>i   s     !B/Q#a&/s   r=   )rB   r=   r>   zsplit z not found in info ())rK   lenrG   r   zipsumrO   keysvaluesr;   RuntimeErrorrP   )rK   rL   rR   r=   
split_namesplit_parts	split_idxsplit_filenames
split_infor5   cr\   s              @r8   _parse_split_inforl   K   s   
 ~ C 
u:>eAh-Kau++c1-KK(1,IX-+2HDQYN2Z(3
%e,h
3J(+J{,CZP_E`(ab(aqt(ab!!B/!BB*/
*;
;'.3CJJL.A
?+,7=)&z2J  #')J  45X#>w.B488HVXCYC^C^C`Baabcdd(^E*
":.
' cs   Fc                 v    [         R                  S[        U 5       S35        [        U [        5      (       a  U eg)zQCall in an exception handler to ignore exceptions, issue a warning, and continue.zHandling webdataset error (z). Ignoring.T)r-   r.   reprr   	TypeError)exns    r8   log_and_continuerq   ~   s1    OO1$s)LIJ#y!!	rJ   c                    U(       a.  [         R                  " U S   5      n[        XT   5      nUS:  a  gO[        X   5      n[        X5      n[        R
                  " U5       n[        R                  " U5      nUR                  5         SSS5        U(       a  UR                  U5      n[        XvU R                  SS5      S9n	U	$ ! , (       d  f       NB= f)zCustom sample decode
* decode and convert PIL Image
* cls byte string label to int
* pass through JSON byte string (if it exists) without parse
r(   r   N)jpgclsr(   )r(   loadsrG   r   ioBytesIOr   openr)   convertdictrP   )
sample	image_key
image_mode
target_keyrA   metaclass_labelimgbdecodeds
             r8   _decoder      s     zz&.)$/*?  &,- 6
%C	CAjjm
 
 kk*% s&**VT2JKGN 
s   $'C
Cc                  p    [        5       n U b  U R                  $ [        R                  R	                  5       $ )z'get dataloader worker seed from pytorch)r   seedr%   utilspytorch_worker_seed)worker_infos    r8   r   r      s/    !#K99((**rJ   c                   ,    \ rS rSr    SS jrS rSrg)detshuffle2   c                 4    Xl         X l        X0l        X@l        g rZ   )bufsizeinitialr   epoch)selfr   r   r   r   s        r8   __init__detshuffle2.__init__   s     #L"LIJrJ   c                 x   [        U R                  [        5      (       a  U R                  R                  nO!U =R                  S-  sl        U R                  nU R                  S:  a  [        5       U-   nOU R                  U-   n[        R                  " U5      n[        XR                  U R                  U5      $ )Nr   r   )r   r   r   valuer   r   randomRandomr   r   r   )r   srcr   r   rngs        r8   rundetshuffle2.run   s    $**k22

(( 

a


yy1}*,u4yy5(--%CCt||SAArJ   )r   r   r   r   N)i  d   r   )rC   rD   rE   rF   r   r   rI   r?   rJ   r8   r   r      s     
		BrJ   r   c                   R   ^  \ rS rSrSr\R                  SSS4U 4S jjrS rSr	U =r
$ )	ResampledShards2   z,An iterable dataset yielding a list of urls.NTr   c                 ,  > [         TU ]  5         [        R                  R	                  U5      nXl        [        U R
                  S   [        5      (       d   eX l        [        R                  " 5       U l        Uc  [        OUU l        X@l        XPl        g)z{Sample shards from the shard list with replacement.

:param urls: a list of URLs as a Python list or brace notation string
r   N)superr   r%   
shardlistsr   urlsr   r    nshardsr   r   r   r   worker_seeddeterministicr   )r   r   r   r   r   r   	__class__s         r8   r   ResampledShards2.__init__   sq     	~~))$/	$))A,,,,,==?2=2E.;*
rJ   c              #      #    [        U R                  [        5      (       a  U R                  R                  nO!U =R                  S-  sl        U R                  nU R                  (       a,  [
        R                  " U R                  5       U-   5      U l        [        U R                  5       HM  nU R                  R                  S[        U R                  5      S-
  5      n[        U R                  U   S9v   MO     g7f)z#Return an iterator over the shards.r   r   )urlN)r   r   r   r   r   r   r   r   r   ranger   randintr`   r   rz   )r   r   _indexs       r8   __iter__ResampledShards2.__iter__   s     djj+..JJ$$E JJ!OJJJE}}T%5%5%7%%?@DHt||$AHH$$QDII(:;E499U+,, %s   C<C>)r   r   r   r   r   r   )rC   rD   rE   rF   __doc__sysmaxsizer   r   rI   __classcell__r   s   @r8   r   r      s'    6
 ,- -rJ   r   c            "          ^  \ rS rSr               SS\S\\   S\S\S\\   S\S\S	\S
\\   S\S\S\S\S\S\\   S\\   4 U 4S jjjr	S r
 SS\\   4S jjrS rS rS rS rS rS S jrS S jrSrU =r$ )!	ReaderWdsi  r/   rB   rK   is_trainingr=   
batch_sizerepeatsr   	class_map	input_keyinput_img_moder~   target_img_modefilename_keysample_shuffle_sizesample_initial_sizec                   > [         TU ]  5         [        c  [        S5      eXl        X@l        X`l        Xpl        Xl        SU l	        U=(       d    [        U l        U=(       d    [        U l        Xl        Xl        Xl        Xl        SU l        ['        U R                  5      U l        [+        X0R(                  5      U l        Ub  XPl        OU R,                  R.                  U l        U(       a  U R.                  (       d  [        S5      eSU l        U	(       a  [3        U	5      U l        SU l        O0 U l        SU l        SU l        [:        R<                  " 5       (       ag  [:        R>                  " 5       (       aM  [:        R@                  " 5       S:  a4  [:        RB                  " 5       U l        [:        R@                  " 5       U l        S U l"        SU l#        Xl$        SU l%        SU l&        SU l'        SU l(        [S        5       U l*        S U l+        g )	NzcPlease install webdataset 0.2.x package `pip install git+https://github.com/webdataset/webdataset`.i  z.JPEGzBInvalid split definition, num_samples not specified in train mode.FTr   r   ),r   r   r%   re   r/   r   r   r   common_seedshard_shuffle_sizeSAMPLE_SHUFFLE_SIZEr   SAMPLE_INITIAL_SIZEr   r   r   r~   r   key_extr9   rL   rl   rj   r=   remap_classr   class_to_idx	dist_rankdist_num_replicasdistis_availableis_initializedget_world_sizeget_rankr   	worker_idr   num_workersglobal_worker_idglobal_num_workers
init_countr   epoch_countds)r   r/   rB   rK   r   r=   r   r   r   r   r   r   r~   r   r   r   r   r   s                    r8   r   ReaderWds.__init__  s   & 	;uw w	&$"%#6#M:M #6#M:M ",$(tyy)	+E99="*#::Dt//!cee  .y 9D#D "D !"4#6#6#8#8T=P=P=RUV=V!]]_DN%)%8%8%:D"   !"#&= rJ   c                 $    XR                   l        g rZ   )r   r   )r   counts     r8   	set_epochReaderWds.set_epochQ  s    !&rJ   r   c                 p    U R                   b  g Ub%  Xl        U R                  U R                  -  U l        g g rZ   )r   r   r   r   )r   r   s     r8   set_loader_cfgReaderWds.set_loader_cfgT  s;     77"*&*&<&<t?O?O&OD# #rJ   c                 X   U R                   c  [        R                  R                  R	                  5       nUb9  Xl         UR
                  U l        UR                  U l        UR                  U l	        U R                  U R                  -  U l        U R                  U R                  -  U R                  -   U l        U R                  R                   Vs/ s H-  n[         R"                  R%                  U R&                  U5      PM/     nn[(        R*                  " U5      /nU R,                  (       a  UR/                  [1        U R2                  U R4                  U R6                  S9U R8                  [(        R:                  " [<        S9[(        R>                  " U R@                  U RB                  [D        RF                  " U R                  5      S9/5        O3UR/                  U R8                  [(        R:                  " [<        S9/5        UR/                  [(        RH                  " [K        [L        U RN                  U RP                  U R                  RR                  S9[<        S9[(        RT                  " U RN                  U RV                  S9/5        [(        RX                  " U6 U l-        gs  snf )z7Lazily initialize worker (in worker processes)
        N)r   r   )handler)r   r   r   )r|   r}   rA   )imagetarget).r   torchr   datar   idr   r   r   r   r   r   r   r   rj   r>   r!   r"   r#   r/   r%   SimpleShardListr   extendr   r   r   r   _split_by_node_and_workertarfile_to_samplesrq   shuffler   r   r   r   mapr   r   r   r   rA   renamer~   DataPipeliner   )r   r   r5   abs_shard_filenamespipelines        r8   
_lazy_initReaderWds._lazy_init^  s    #++**::<K&#. !,#.#3#3 #.#:#: &*&<&<t?O?O&OD#$(NNT5E5E$E$VD! DH??C\C\]C\arww||DIIq9C\]''(;<=OO++))**
 ..&&/?@ 44 44d&6&67   OO..&&/?@ 
 	GG"nn#22"oo77	 ) JJT^^DOOD
 	 ""H-K ^s   4J'c              #      #    U R                   S:  a-  [        XR                  S U R                   5       H  nUv   M	     g U H  nUv   M	     g 7fNr   )r   r   r   )r   r   ss      r8   r   #ReaderWds._split_by_node_and_worker  sI     ""Q&C!6!6d>U>UV W  s   AAc                 `   U R                   [        U R                  U R                  5      -  nU R                  (       d  U R                  S:  a  [
        R                  " U5      nU R                  (       a/  [
        R                  " XR                  -  5      U R                  -  n[        U5      $ r   )	r=   maxr   r   r   mathceilr   rG   )r   num_worker_sampless     r8   _num_samples_per_worker!ReaderWds._num_samples_per_worker  s    !--D4K4KTMcMc0ddt559!%+=!>!%+=+O!PSWSbSb!b%&&rJ   c              #   t  #    U R                   c  U R                  5         U R                  5       nU R                  (       d  U R                  S:  a  U R                   R                  U5      nOU R                   nSnU H6  nUS   nU R                  (       a  U R                  U   nUS   U4v   US-  nM8     g 7f)Nr   r   r   r   )r   r   r   r   r   
with_epochr   r   )r   r   r   ir{   r   s         r8   r   ReaderWds.__iter__  s     77?OO!99;t559
 ##$67BBFH%F**62/6))FA s   B6B8c                 @    U R                  5       U R                  -  nU$ rZ   )r   r   )r   r=   s     r8   __len__ReaderWds.__len__  s!    224t7G7GGrJ   c                      S5       e)NzNot supportedr?   )r   r   basenameabsolutes       r8   	_filenameReaderWds._filename  s    %o%urJ   c                 B   U R                   c  U R                  5         / nU R                    Ho  nU R                  U;   a  X@R                     nO SU;   a  US   U R                  -   nO S5       eUR	                  U5        [        U5      U R                  :  d  Mn    U$    U$ )z/Return all filenames in dataset, overrides base__key__zNo supported name field present)r   r   r   r   r$   r`   r=   )r   r  r  r0   r{   rB   s         r8   r>   ReaderWds.filenames  s    77?OOggF  F*//0f$i(4<<7???uLL5zT---  rJ   )r   r   r   r   r   r   r   r   r   r   rL   r   r   r   r   r   r=   r   r   r   r/   r   r   r   rj   r~   r   r   r   )NtrainFNr   r   *   Nzjpg;png;webpRGBrt   r   filenameNNrZ   )FF)rC   rD   rE   rF   r    r
   boolrG   rz   r   r   r   r   r   r   r   r  r	  r>   rI   r   r   s   @r8   r   r     sG    #'  %)-(,+"'##% *1515#HH 3-H 	H
 H "#H H H H  ~H H  H H !H H  "*#!H" "*##H HT'
 *.P!#P3.j'0& rJ   r   ))z
_info.jsonz	info.json)rs   r  rt   r   )Fr   rv   r(   loggingr   r!   r   r   dataclassesr   	functoolsr   	itertoolsr   typingr   r   r   r	   r
   r   r   torch.distributeddistributedr   r*   PILr   torch.utils.datar   r   r   
webdatasetr%   webdataset.filtersr   r   webdataset.shardlistsr   webdataset.tariteratorsr   r   r   r   ImportErrorr   r   readerr   shared_countr   	getLoggerrC   r-   rG   environrP   r   r   r9   r;   r    rl   rq   r   r   PipelineStager   r   r   r?   rJ   r8   <module>r&     sP   
    	  
 !   = =      F F51bb
 &  %


H
%"**..);TBC "**..);TBC 0   0S 0 0f !H+ ?Bc'' B> K)- )-XL LU  
CKs    E 
EE