
    x-j                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZ	d dl
Z
d dl
mZ d dlmZmZ d dlmZ d dlmZ d dlmZ ddlmZmZmZ d	d
lmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z* da+d Z, ej-        e,            G d d          Z. G d de.          Z/ G d de.          Z0dS )    N)profiler)_current_expected_place_set_expected_place)datatype_to_vartype)	benchmark)in_profiler_mode   )corein_dynamic_modein_pir_mode   )MP_STATUS_CHECK_INTERVALCleanupFuncRegistrar_set_SIGCHLD_handler   )_InfiniteIterableSampler)default_collate_fndefault_convert_fn)_flatten_batch_restore_batch)_DatasetKind_IterableDatasetStopIteration_ResumeIteration_worker_loop_WorkerExceptionc                  ^    t           %	 t                                            b d S #  Y d S xY wd S N)_loader__del__     d/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/io/dataloader/dataloader_iter.py_clear_loaderr#   D   s<    	OO	DD s   % *c                   L    e Zd ZdZd Zed             Zd Zd Zd Z	d Z
d Zd	S )
_DataLoaderIterBasez
    Iterator implement of DataLoader, will load and feed mini-batch
    data by setting in given dataloader.

    Args:
        loader(instance of DataLoader): instance of `paddle.io.DataLoader`
    c                    |j         | _        |j        pg | _        |j        | _        |j        | _        |j        | _	        |j
        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        dk    r|j        nt2          | _        |j        | _        |j        | _        |j        | _         tC          | j"                  | _#        | j        r|j$        ptJ          | _&        n|j$        ptN          | _&        d | _(        d | _)        tU          j+                    | _,        d S )Nr   )-dataset_dataset	feed_list
_feed_listplaces_placesreturn_list_return_listbatch_sampler_batch_sampler	drop_last
_drop_lastauto_collate_batch_auto_collate_batchnum_workers_num_workersuse_buffer_reader_use_buffer_readerreader_buffer_size_reader_buffer_sizeprefetch_factor_prefetch_factoruse_shared_memory_use_shared_memorytimeoutr   _timeoutworker_init_fn_worker_init_fndataset_kind_dataset_kind
pin_memory_pin_memoryiter_index_sampler_sampler_iter
collate_fnr   _collate_fnr   _blocking_queue_thread	threadingEvent_thread_done_event)selfloaders     r"   __init__z_DataLoaderIterBase.__init__Z   s0    *0b}".$2 *#)#< "."(":#)#<  & 6"(":$nq00FNN6N 	  &4#0!,!$"566# 	G%0F4FD%0F4FD  $"+/"3"3r!   c                     | j         r| j        S | j        t          j        k    r.t          t          t          | j                                      S t          | j        d          S Nr   )
r4   r0   rD   r   MAPlistrangelenr(   r   rQ   s    r"   rH   z"_DataLoaderIterBase._index_sampler}   sZ    # 	B&&!\%555E#dm"4"455666/qAAAr!   c                     | S r   r    rZ   s    r"   __iter__z_DataLoaderIterBase.__iter__   s    r!   c                      t          d          )Nz*Should implement `__next__` for a iterator)NotImplementedErrorrZ   s    r"   __next__z_DataLoaderIterBase.__next__   s    !"NOOOr!   c                 *    t          | j                  S r   )rY   r0   rZ   s    r"   __len__z_DataLoaderIterBase.__len__   s    4&'''r!   c                 |    | j                                          | j        r| j                                         d S d S r   )rP   setrL   closerZ   s    r"   _exit_thread_expectedlyz+_DataLoaderIterBase._exit_thread_expectedly   sG    ##%%% 	) &&(((((	) 	)r!   c                 |    | j                                          | j        r| j                                         d S d S r   )rP   rc   rL   killrZ   s    r"   _exit_thread_unexpectedlyz-_DataLoaderIterBase._exit_thread_unexpectedly   sG    ##%%% 	( %%'''''	( 	(r!   N)__name__
__module____qualname____doc__rS   propertyrH   r\   r_   ra   re   rh   r    r!   r"   r%   r%   Q   s         !4 !4 !4F B B XB  P P P( ( () ) )
( ( ( ( (r!   r%   c                   F     e Zd ZdZ fdZd Zd Zd Zd Zd Z	d Z
 xZS )	_DataLoaderIterSingleProcesszg
    Single process implement of DataLoaderIter, loading data from
    loader.data in main process
    c                    t                                          |           t          j        | j        | j        | j        | j        | j                  | _	        g | _
        | j        t          | j                  z  | _        d| _        	 |                                  n## t"          $ r |                                   w xY w| ad S NF)superrS   r   create_fetcherrD   r(   r4   rK   r2   _dataset_fetcher_structure_infosr<   rY   r,   _blocking_queue_capacity	_shutdown_init_thread	Exception_try_shutdown_allr   )rQ   rR   	__class__s     r"   rS   z%_DataLoaderIterSingleProcess.__init__   s        , ;M$O!
 !
 !#
 )-(=LA
 A
 )
% 	 	 	 	""$$$	
 s   	B  B>c                    d | j         D             | _        d | j         D             | _        t                      r-d | j         D             | _        d | j         D             | _        n,d | j         D             | _        d | j         D             | _        t          j        t          j                    | j	        t          | j                  dk              | _        t          j        | j        | j        | j        | j        | j        | j        | j        d| j        | j        
  
        | _        t%          j        | j        t+                      f	          | _        d| j        _        | j                                         d S )
Nc                     g | ]	}|j         
S r    name.0vs     r"   
<listcomp>z=_DataLoaderIterSingleProcess._init_thread.<locals>.<listcomp>       ;;;a16;;;r!   c                     g | ]	}|j         
S r    shaper   s     r"   r   z=_DataLoaderIterSingleProcess._init_thread.<locals>.<listcomp>       999A999r!   c                     g | ]}d S Fr    r   s     r"   r   z=_DataLoaderIterSingleProcess._init_thread.<locals>.<listcomp>       $D$D$DqU$D$D$Dr!   c                 2    g | ]}t           |j                 S r    r   dtyper   s     r"   r   z=_DataLoaderIterSingleProcess._init_thread.<locals>.<listcomp>   -       12#AG,  r!   c                 @    g | ]}|j                                         S r    descneed_check_feedr   s     r"   r   z=_DataLoaderIterSingleProcess._init_thread.<locals>.<listcomp>   3     % % %-.&&((% % %r!   c                     g | ]	}|j         
S r    r   r   s     r"   r   z=_DataLoaderIterSingleProcess._init_thread.<locals>.<listcomp>       ===AG===r!   r   Ttargetargs)r*   
_var_names_shapesr   _need_check_feed_dtypesr
    init_dense_tensor_blocking_queueVariablerv   rY   r,   rL   create_py_readerr8   rF   r:   _readerrN   Thread_thread_loopr   rM   daemonstartrZ   s    r"   rx   z)_DataLoaderIterSingleProcess._init_thread   s   ;;4?;;;99999== 		>$D$DDO$D$D$DD! 6:o  DLL% %26/% % %D! >=T_===DL#DMOO)! 
  

 , OLL!L#$
 
 !'$,C,E,E+G
 
 
 #r!   c                    t          j        dt          t          |                     z              t	          |           | j                                        s	 t          | j                  }| j	        
                    || j                  }n%# t          $ r |                                  Y d S w xY w|| j                                        rnt          |          \  }}| j                            |           | j                                        rnY	 t          j                    }|D ]}t#          |t$          j                  r'|                                                                }nVt#          |t           j                  s<t          j                    }|                    |t          j                               |}|                    |           | j                                        rnz	 | j                            |           n#  |                                  Y nxY wn(# t6          $ r}|                                  |d }~ww xY w| j                                        |                                  d S )NDataloader_)r
   set_current_thread_namestridr   rP   is_setnextrI   rt   fetchStopIterationre   r   ru   appendDenseTensorArray
isinstancepaddleTensorvalue
get_tensorDenseTensorrc   CPUPlacerL   pushry   rh   )	rQ   legacy_expected_placeindicesbatch	structurearrayslottmpes	            r"   r   z)_DataLoaderIterSingleProcess._thread_loop   s    	$]SD]]%BCCC1222)0022 /	t122
 -33T4  !   ,,... } 7 > > @ @}  .e44E9!((333&--// -//! ' 'D!$66 ##zz||6688'd.>?? #".00dmoo666"LL&&&&*1133 3(--e444430022222   ..000[ )0022 /	b 	$$&&&&&sC   4B B32B3CH1 :H H1 H-+H1 1
I;IIc                     t                      r9t          j        dt          j        j                  }|                                 	 t                                                      t                                                       t                      ret          j                             j                                        d                   }t          | j                            d                    }n݉ j        r j                                        }t'          t)          |                    D ]}||                                         ||<     fdt'          t)           j                            D             }d t/          ||          D             }t)           j                  dk    r|d         }n j                                        }t                                                       |t                      r|                                 S S # t6          $ r/  j                                                                            w xY w# t                      r|                                 w w xY w)Nro   r   
event_typer   c                 D    g | ]}j                             d           S r   ru   popr   _rQ   s     r"   r   z9_DataLoaderIterSingleProcess.__next__.<locals>.<listcomp><  :        -11!44  r!   c                 4    g | ]\  }}t          ||          S r    r   r   dss      r"   r   z9_DataLoaderIterSingleProcess.__next__.<locals>.<listcomp>@  &    PPPTQN1a00PPPr!   r   )r   r   RecordEventTracerEventType
Dataloaderbeginr   check_if_need_recordbefore_readerr   r
   eagerread_next_tensor_listr   read_next_listr   ru   r   r.   rX   rY   _move_to_listr,   zip	read_nextafter_readerendr   shutdownrz   rQ   trace_eventdataistructss   `    r"   r_   z%_DataLoaderIterSingleProcess.__next__'  ss    	 ".3#3>  K #	"KK,,T222KK%%'''   4z77L//11!4  &dD,A,E,Ea,H,HII $ 4<6688D"3t99-- : :"&q'"7"7"9"9Q   !&s4<'8'8!9!9  G QPSw=O=OPPPD 4<((A--#Aw<1133DKK$$&&&  !! "!!!!"  	 	 	L!!###""$$$	
  !! "!!!!"s   
F2H   9II %Jc                 @   | j         r| j                                         t          d          D ]1}| j                                         rt          j        d           0 n3| j         t          j                    ur| j         	                                 d | _         d S d S )Nr	   r   )
rM   rP   rc   rX   is_alivetimesleeprN   current_threadjoinrQ   r   s     r"   _shutdown_threadz-_DataLoaderIterSingleProcess._shutdown_threadS  s    < 	 #''))) 1XX ( (<((** JqMMMME<y'?'A'AAAL%%'''DLLL	  	 r!   c                     | j         sQ	 | j        r | j                                         d | _        |                                  d| _         d S # d| _         w xY wd S )NT)rw   rL   rd   r   rZ   s    r"   rz   z._DataLoaderIterSingleProcess._try_shutdown_allc  sw    ~ 	&& ' 0(..000+/D( %%'''!%%%%%	& 	&s   ;A 	Ac                 .    |                                   d S r   rz   rZ   s    r"   r   z$_DataLoaderIterSingleProcess.__del__r           r!   )ri   rj   rk   rl   rS   rx   r   r_   r   rz   r   __classcell__r{   s   @r"   ro   ro      s         
" " " " "H$ $ $L:' :' :'x*" *" *"X      & & &! ! ! ! ! ! !r!   ro   c                   p     e Zd Z fdZd Zd Zd Zd ZddZdd	Z	d
 Z
d Zd Zd Zd Zd Zd Z xZS )_DataLoaderIterMultiProcessc                     t                                          |           |j        | _        d| _        | j        dk    sJ d| j         d            d | _        d| _        d| _        d| _        i | _	        g | _
        | j        t          | j        t          | j                            z  | _        t!          j                    | _        t&          j                            dt,          j                  | _        t2          j                            dd          dv rF	 dt          | j        d                   z  | _        n)#  d| _        t=          j        d	           Y n
xY wd| _        | j        d
z  | j        z  | _         d| _!        | "                                 tG          | j                  D ]}| $                                 	 | %                                 d S # tL          $ r | '                                  w xY w)Nr   z-Multi-process DataLoader invalid num_workers())lowhighFLAGS_use_shm_cacheF)r   1TTruetruer	   zUSetting the shm cache buffer size to 0, equivalent to not using the shm cache policy.r   )(rr   rS   _persistent_workers_resume_worker_cntr6   _data_queue	_send_idx	_rcvd_idx_batches_outstanding_task_infosru   r<   maxrY   r,   _outstanding_capacityrN   Lock_thread_locknprandomrandintsysmaxsize
_base_seedosenvirongetr(   _worker_shm_buffer_sizewarningswarn_main_thread_shm_buffer_sizerw   _init_workersrX   _try_put_indicesrx   ry   rz   )rQ   rR   r   r{   s      r"   rS   z$_DataLoaderIterMultiProcess.__init__w  s$      #)#= "# 1$$$PD<MPPP %$$
  
 $%! " &*%:Ss4<00>
 >
 &
"
 &N,,)+++DD
 :>>/77 <
 
 
05T]1=M9N9N/N,,/0,k     ,-D()Q.1BB 	) t122 	$ 	$A!!####	 	 	 	""$$$	s   "E   EG  G=c                    ddl m} g | _        g | _        g | _        t          j        t          | j                            | _	        |
                                | _        |                                | _        t          j                    | _        t          | j                  D ]}|
                                }|                                 | j                            |           |                    t&          | j        | j        || j        | j        | j        | j        | j        | j        || j        | j        | j        | j        f          }d|_        |                                 | j                            |           | j                            d           t?          j         tC          |           tE          d | j        D                                  tG                       d S )Nr   )multiprocessingr   Tc              3   $   K   | ]}|j         V  d S r   )pidr   ws     r"   	<genexpr>z<_DataLoaderIterMultiProcess._init_workers.<locals>.<genexpr>  s$      .L.Lqu.L.L.L.L.L.Lr!   )$paddle.incubater  _workers_worker_status_indices_queues	itertoolscyclerX   r6   _workers_idx_cycleQueuer   rO   _workers_done_eventrN   rP   cancel_join_threadr   Processr   r(   rD   r4   rK   r2   rB   r>   r  r  r   r   r
   _set_process_pidsr   tupler   )rQ   r  r   indices_queueworkers        r"   r  z)_DataLoaderIterMultiProcess._init_workers  s   333333  !"+/%8I2J2J"K"K +0022 $3#8#8#:#: "+/"3"3t()) 	- 	-A+1133M,,... ''666$,,#M&!$,,$O(%+O0 -  F& !FMLLNNNM  (((&&t,,,,r$xx.L.Ldm.L.L.L)L)LMMMr!   c                     | j         W	 	 | j                                          n:#  | j                                          | j                                          Y d S xY wVd S r   )r   
get_nowaitr(  rd   rZ   s    r"   _clear_and_remove_data_queuez8_DataLoaderIterMultiProcess._clear_and_remove_data_queue  st    '$//1111$77999$**,,,EE ('s	   $ 4Ac                 ^   d | j         D             | _        d | j         D             | _        t                      r-d | j         D             | _        d | j         D             | _        n,d | j         D             | _        d | j         D             | _        t          j        t          j                    | j	        t          | j                  dk              | _        t          j        | j                   t          j        | j        | j        | j        | j        | j        | j        | j        d| j        | j        
  
        | _        t)          j                    | _        t)          j        | j        t3                      f	          | _        d| j        _        | j                                         d S )
Nc                     g | ]	}|j         
S r    r~   r   s     r"   r   z<_DataLoaderIterMultiProcess._init_thread.<locals>.<listcomp>  r   r!   c                     g | ]	}|j         
S r    r   r   s     r"   r   z<_DataLoaderIterMultiProcess._init_thread.<locals>.<listcomp>  r   r!   c                     g | ]}d S r   r    r   s     r"   r   z<_DataLoaderIterMultiProcess._init_thread.<locals>.<listcomp>  r   r!   c                 2    g | ]}t           |j                 S r    r   r   s     r"   r   z<_DataLoaderIterMultiProcess._init_thread.<locals>.<listcomp>  r   r!   c                 @    g | ]}|j                                         S r    r   r   s     r"   r   z<_DataLoaderIterMultiProcess._init_thread.<locals>.<listcomp>  r   r!   c                     g | ]	}|j         
S r    r   r   s     r"   r   z<_DataLoaderIterMultiProcess._init_thread.<locals>.<listcomp>  r   r!   r   Tr   )r*   r   r   r   r   r   r
   r   r   r  rY   r,   rL   (_set_max_memory_map_allocation_pool_sizer  r   r8   rF   r:   r   rN   rO   rP   r   r   r   rM   r   r   rZ   s    r"   rx   z(_DataLoaderIterMultiProcess._init_thread  s   ;;4?;;;99999== 		>$D$DDO$D$D$DD! 6:o  DLL% %26/% % %D! >=T_===DL#DMOOT7T\9J9JQ9N 
  
 	5-	
 	
 	
 , OLL!L#$
 
 #,/"3"3 '$,C,E,E+G
 
 
 #r!   c                    | j         5  | j        | _        t          | j                  D ]>}| j        |                             t                                 | xj        dz  c_        ?	 d d d            n# 1 swxY w Y   | j        dk    rt          j	        d           | j        dk    | j
                                        t          | j                  k    rt                      r=t          j                            | j                                        d                   }n:| j        r| j                                         n| j                                        }| j
                                        t          | j                  k    d| _        d| _        d| _        i | _        g | _        dg| j        z  | _        t7          | j                  | _        t          | j                  D ]}|                                  d S )Nr   r   g      ?T) r  r6   r   rX   r"  putr   r  r   r   rL   sizerY   r,   r   r
   r   r   r   r   r.   r   r  r  r  ru   r!  rG   rH   rI   r  r  )rQ   	worker_idr   r   s       r"   _resetz"_DataLoaderIterMultiProcess._reset"  s=     	/ 	/&*&7D#"4#455 / /	$Y/334D4F4FGGG))Q.)))/	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ %))JsOOO %)) "''))S->->>>   4z77L//11!4  $ 4L//1111<1133D "''))S->->>> $%! "  $ft'88 "$"566t122 	$ 	$A!!####	$ 	$s   A A55A9<A9Fc                     |t          | j                  k     rB| j        |         s	| j        r0|r0| j        |                             d            d| j        |<   d S d S d S d S rq   )rY   r!  r   r"  r:  )rQ   r<  r   s      r"   _shutdown_workerz,_DataLoaderIterMultiProcess._shutdown_workerM  s    s4.////	* 0' 0  0
  +//555-2D	*** 0/////r!   Nc                 T   | j         s	 |                                  |                                  | j                                         t          | j                  D ]}|                     |d           | j         sQ| j        D ]}|	                    |           | j
        D ]*}|                                 |                                 +t          j        t          |                      d| _         d S # t          j        t          |                      d| _         w xY wd S )NT)r   )rw   re   r0  r'  rc   rX   r6   r?  r   r   r"  r(  rd   r
   _erase_process_pidsr   )rQ   r?   r   r  qs        r"   rz   z-_DataLoaderIterMultiProcess._try_shutdown_allV  s:   ~ 	&&,,...11333
 (,,...t011 < <A))!d);;;;~ "!] ( (w!1 " ",,...				(D222!% (D222!%%%%%)	& 	&s   CC; ;*D%c                    t          j        dt          t          |                     z              t	          |           | j                                        s|                                 }| j                                        s||                                  nt          |t                    r| j        dk    sJ | xj        dz  c_        	 t          j                    }| j        r|D ]}|                    |           n|D ]}t          |t          j                  r|                                }nVt          |t           j                  s<t          j                    }|                    |t          j                               |}|                    |           | j                            |          s| j                                         n(# t0          $ r}|                                  |d }~ww xY w| xj        dz  c_        n# | xj        dz  c_        w xY w| j                                        d S d S )Nr   r   r   )r
   r   r   r   r   rP   r   	_get_datare   r   r   r   r   r>   r   r   r   r   r   rc   r   rL   r   rd   ry   rh   r  )rQ   r   r   r   tensorr   r   r   s           r"   r   z(_DataLoaderIterMultiProcess._thread_loopm  s]    	$]SD]]%BCCC1222)0022 "	,NN$$E*1133  ,=002222!%)9:: !#6:::://14// , $ 5 7 72 3*/ 5 5 %V 4 4 4 45
 ). 3 3#-dFM#B#B !/+/??+<+<DD)3D$:J)K)K !/*.*:*<*<C$'GGD$-//$B$B$B+.D %T 2 2 2 2#388?? 9 066888$      66888  !+!+E )0022 "	, "	, "	, "	, "	,s+   DG H 
H'G==HH H(c                    | j                                         s| j        t          j        k    r| j        | j        k     rv| j        | j                 }t          |          dk    s| j	        |d                  rnc| j        | j        = | xj        dz  c_        | xj
        dz  c_
        | j        | j        k     v| j        s| j
        t          | j                  k     rd S | j        | j        v rjt          | j        | j                           dk    rG| j                            | j                  }| j                            |d                    |d         S 	 | j                            | j                  }| j        t          j        k    rlt'          |t(                    rW| j        rd| j	        |j        <   n*|                     |j                   | xj
        dz  c_
        |                                  |\  }}}t'          |t0                    r|||S t'          |t2                    r(|                                  |                                 || j        k    r-|| j        v r| j        |= | j                            |           |S | j        |xx         ||fz  cc<   # t8          $ rS}| j                                         rY d }~g }t;          | j                  D ]P\  }}	| j	        |         r>|	                                s*|                    |	           |                     |           Qt          |          dk    r`|                                  d                     d |D                       }
tC          j"        d	t          |           d
|
            Y d }~d S t'          |tF          tH          j%        f          rY d }~|                                  tC          j&        d| d           |d }~ww xY wd S )Nr	   r   r   r   )r?   Fz, c              3   >   K   | ]}t          |j                  V  d S r   )r   r  r  s     r"   r  z8_DataLoaderIterMultiProcess._get_data.<locals>.<genexpr>  s*      $H$HASZZ$H$H$H$H$H$Hr!   zDataLoader z" workers exit unexpectedly, pids: z DataLoader reader thread failed(z*) to read data from workers' result queue.)'rP   r   rD   r   ITERr  r  r  rY   r!  r  r   r,   r   ru   r   r   r  r@   r   r   r<  r?  r  r   r   rh   reraisery   	enumerater   r   r   loggingwarningIOErrorqueueEmptyerror)rQ   infor   idxr   r   r   failed_workersr   r  pidss              r"   rD  z%_DataLoaderIterMultiProcess._get_data  sr   )0022 x	 !\%666nt~55+DN;D4yyA~~)<T!W)E~(8NNa'NN--2-- nt~55&  3 (  4s4<7H7HHH#'4 $"222(899Q>>'++DN;;%,,T!W555AwN '++DM+BBB %):::z7@ @: / 7>C+DN;;--dn===11Q611))+++(,%UI s$455!)Je%566 $22444MMOOO$.((d... ,S1)00;;; L$S)))eY-??)))I    *1133 HHHH "$%dm44 1 1DAq*1- 1ajjll 1&--a000--a000~&&**2244499$H$H$H$H$HHHDO(c.&9&9 ( (!%( (   FFFFF a'5;!788 HHHH..000-q - - -   =ix	 x	s+    J% %P0O>CO>)!O>.O>>Pc                    | j         | j        k    s
J d            | j        5  	 t          | j                  }n# t
          $ r Y d d d            d S w xY wt          | j                  D ]%}t          | j                  }| j	        |         r n&	 d d d            d S | j
        |                             | j        |f           |f| j        | j        <   | xj         dz  c_         | xj        dz  c_        d d d            d S # 1 swxY w Y   d S )Nz'too many indices have been put to queuer   )r  r  r  r   rI   r   rX   r6   r%  r!  r"  r:  r  r  )rQ   r   r   
worker_idxs       r"   r  z,_DataLoaderIterMultiProcess._try_put_indices  s   (D,FFFF5 GFF  	  	 t122    		  	  	  	  	  	  	  	  4,--  !$"9::
&z2 E 	  	  	  	  	  	  	  	   ,00$.'1JKKK0:}DT^,%%*%%NNaNN!	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	 s8   D9D
ADA>DADDDc                 .    |                                   d S r   r   rZ   s    r"   r   z#_DataLoaderIterMultiProcess.__del__4  r   r!   c                 0    |                      d           d S rU   r   rZ   s    r"   _shutdown_on_exitz-_DataLoaderIterMultiProcess._shutdown_on_exit7  s    q!!!!!r!   c                     t                      r9t          j        dt          j        j                  }|                                 	 t                                                      t                                                        j	        t           j                  k     r@ j        rt           j                                          j                                         t%                      ret&          j                             j                                        d                   }t1          | j                            d                    }n݉ j        r j                                        }t9          t          |                    D ]}||                                         ||<     fdt9          t           j                            D             }d t=          ||          D             }t           j                  dk    r|d         }n j                                        }                                   t                      !                                 |t                      r|"                                 S S # t          $ r6  j        s- j        #                                  $                                  w xY w# t                      r|"                                 w w xY w)Nr   r   r   c                 D    g | ]}j                             d           S r   r   r   s     r"   r   z8_DataLoaderIterMultiProcess.__next__.<locals>.<listcomp>\  r   r!   c                 4    g | ]\  }}t          ||          S r    r   r   s      r"   r   z8_DataLoaderIterMultiProcess.__next__.<locals>.<listcomp>`  r   r!   r   )%r   r   r   r   r   r   r   r   r   r  rY   r,   r   r   rP   rc   rL   rd   r   r
   r   r   r   r   r   ru   r   r.   rX   r   r   r   _on_output_batchr   r   r   rz   r   s   `    r"   r_   z$_DataLoaderIterMultiProcess.__next__:  s    	 ".2#3>  K 1	"KK,,T222KK%%''' (3t|+<+<<<+ 1''+//111(..000   4z77L//11!4  &dD,A,E,Ea,H,HII$ 4<6688D"3t99-- : :"&q'"7"7"9"9Q   !&s4<'8'8!9!9  G QPSw=O=OPPPD 4<((A--#Aw<1133D!!###KK$$&&&  !! "!!!!"  	 	 	+ )%%'''&&(((		  !! "!!!!"s   
H#J A KK %K9c                     t          t          | j                            D ]&}| xj        dz  c_        |                                  'd S rU   )rX   rY   r,   r  r  r   s     r"   r]  z,_DataLoaderIterMultiProcess._on_output_batcht  sV    s4<(()) 	$ 	$A%%*%%!!####	$ 	$r!   r   r   )ri   rj   rk   rS   r  r0  rx   r=  r?  rz   r   rD  r  r   rY  r_   r]  r   r   s   @r"   r   r   v  s       F F F F FP. . .`  ' ' 'R)$ )$ )$V3 3 3 3& & & &.+, +, +,Zy y yv     >! ! !" " "8" 8" 8"t$ $ $ $ $ $ $r!   r   )1r#  rK  r  rN  r  rN   r   r  numpyr	  r   r   paddle.base.frameworkr   r   paddle.pir.corer   paddle.profiler.timerr   paddle.profiler.utilsr   	frameworkr
   r   r   multiprocess_utilsr   r   r   r/   r   collater   r   flatr   r   r-  r   r   r   r   r   r   r#   registerr%   ro   r   r    r!   r"   <module>ri     st        				  



                  N N N N N N N N / / / / / / + + + + + + 2 2 2 2 2 2 ; ; ; ; ; ; ; ; ; ;         
 4 3 3 3 3 3 ; ; ; ; ; ; ; ; 0 0 0 0 0 0 0 0             .      m , , ,G( G( G( G( G( G( G( G(TX! X! X! X! X!#6 X! X! X!vA$ A$ A$ A$ A$"5 A$ A$ A$ A$ A$r!   