x-j#ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z gZ ejdZejeejs e jZejedddZdZeedZddZd Zd ejfd Zejfd ZddZdS)N~z.cachepaddledatasetc tjtdS#t$r!}|jtjkrYd}~dSd}~wwxYwN)osmakedirs DATA_HOMEOSErrorerrnoEEXIST)pathexcs U/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/dataset/common.py must_mkdirsr6sb I  9 $ $  % $ $ $ $ $s AAActj}t|dtfddD]}|||S)Nrbc.dS)N)read)fsrzmd5file..DsaffTllr)hashlibmd5openiterupdateclose hexdigest)fnamehash_md5chunkrs @rmd5filer$Ast{}}H UDA****C00GGIII     rc,tjd|}t|tr'tjd|}t jt|}t j |st j |t j||| ddn|}t j |rt||kr|Sd}d}t j |rt||ksst j |r3tjdt|d|d ||kr|d z }nt!d |d |tjd |d|dtjd t#jd|dd5}|jd} | Ct+|d5} t-j|j| dddn #1swxYwYnt+|d5} d} t3| } | | z d z} | dkr| dznd } d}t4jj| d}|| D]P}| ||d z }||i|| zdkr||Q dddn #1swxYwYdddn #1swxYwYn#t@$r }Yd}~jd}~wwxYwt j |_t||kstjdtj!"|S)Nz^[a-zA-Z0-9_/\-]+$z^(?:(?!\.\.)[a-zA-Z0-9_/\.-])+$/rzfile z md5  zCannot download z within retry limit z Cache file z not found, downloading z zBegin to download GETT)timeoutfollow_redirectszcontent-lengthwbritem)name) chunk_sizez Download finished )#rematchgroup isinstancestrrrjoinr existsr splitr$sysstderrwrite RuntimeErrorhttpxstreamheadersgetrshutil copyfileobjrawintrhapi progressbar ProgressBar iter_bytesr Exceptionstdoutflush)url module_namemd5sum save_namedirnamefilenameretry retry_limitr total_lengthrr2 total_iter log_interval log_indexbardataes rdownloadr^Js(0+>>DDFFK)S!!H 0)  %'' gll9k22G 7>>' " " Gw||y'83##iH w~~hGH$5$5$?$? EKw~~h''+GH,=,=,G,G 7>>( # # J J  HWX%6%6HHfHHH I I I ;   QJEEI3IIKII   D( D DC D D D    ./// sD4 6 y}}-=>> 'h--5*15!444555555555555555h--6%) '*<'8'8 %1J%>%B 0:RJ",,Q%%& $k5AA&VB%&LLJL$G$G66DGGDMMM%NIJJy"555(<71<< # 9 5 5 5 6666666666666666 6 6 6 6 6 6 6 6 6 6 6 6 6 6 64    HHHH Sw~~h''+GH,=,=,G,GXJ,---J OsN 5-M?"J = M? J M?J M?%B6M( M?(M, ,M?/M, 0M?3 N ?NN NN N N cdttjDD]Q}dttjd|vr)tjd|RdS)Nc<g|]}|d|S)__) startswith).0xs r zfetch_all..s9all4.@.@ rfetchzpaddle.dataset.)dirrr importlib import_modulerf)rOs r fetch_allrjsv~&&MM  c  #$Ck$C$C D D      #$Ck$C$C D D J J L L L MMrz %05d.picklect|stdg}d}t|D]g\}}||||krG||zdkr>t ||zd5}|||g}|dz }dddn #1swxYwYh|r:t ||zd5}|||ddddS#1swxYwYdSdS)a you can call the function as: split(paddle.dataset.cifar.train10(), line_count=1000, suffix="imikolov-train-%05d.pickle") the output files as: |-imikolov-train-00000.pickle |-imikolov-train-00001.pickle |- ... |-imikolov-train-00480.pickle :param reader: is a reader creator :param line_count: line count for each file :param suffix: the suffix for the output files, should contain "%d" means the id for each file. Default is "%05d.pickle" :param dumper: is a callable function that dump object to file, this function will be called as dumper(obj, f) and obj is the object will be dumped, f is a file object. Default is cPickle.dump. zdumper should be callable.rwr*N)callable TypeError enumerateappendr) reader line_countsuffixdumperlinesindex_fidrs rr:r:s, F  64555 EG&&((##1 Q ??q:~22fw&,, ua   1                  &7"C ( ( A F5!                     s$6BB B 7 CCCcfd}|S)a Create a reader that yield element from the given files, select a file set according trainer count and trainer_id :param files_pattern: the files which generating by split(...) :param trainer_count: total trainer count :param trainer_id: the trainer rank id :param loader: is a callable function that load object from file, this function will be called as loader(f) and f is a file object. Default is cPickle.load c3Ktstdtj}|g}t |D]5\}}|z kr't d|||6|D]=}t|d5}|}|Ed{Vdddn #1swxYwY>dS)Nzloader should be callable.z append file: rV)rmrnglobsortroprintrpr) file_list my_file_listidxfnrru files_patternloader trainer_count trainer_ids rrqz$cluster_files_reader..readers@ :899 9Im,,   ++ ( (GC]"j00*b**+++##B''' ! !Bb# !!q          ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s%CC C )rrrrrqs```` rcluster_files_readerrs5 ! ! ! ! ! ! ! ! MrTc|r!tj|r|S|r&tjj|||St|d)Nz& not exists and auto download disabled)rrr9rrcommonr^ ValueError)rrNrrOr^s r_check_exists_and_downloadrsa t$$ J~$--c;DDDDHHHIIIrr)T)r r{rrhrpickler3rCr;tempfiler?rpaddle.dataset__all__r expanduserHOMEaccessW_OK gettempdirr8r rr$r^rjdumpr:loadrrrrrrs     w#ryrw! 8  D GLLx9 = =  I   AAAAHMMM&36;####N6<[BJJJJJJr