
    x-j8                       d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl	Z
d dlZddlmZ ddlmZmZmZ dd	lmZmZ dd
lmZ erd dlmZ  G d d          Z G d d          Z G d d          Z G d d          Zdad$dZ G d d          Z G d d          Z dZ!dZ"dZ#dZ$dZ%dZ& e
j'        e
j(                  j)        d z  dz  Z*d!Z+d%d"Z,	 d&d#Z-dS )'    )annotationsN)TYPE_CHECKINGAny   )core   )MP_STATUS_CHECK_INTERVALCleanupFuncRegistrar_cleanup_mmap   )_IterableDatasetFetcher_MapDatasetFetcher)_flatten_batch)Datasetc                      e Zd Zd ZdS )_IterableDatasetStopIterationc                    || _         d S N)	worker_id)selfr   s     [/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/io/dataloader/worker.py__init__z&_IterableDatasetStopIteration.__init__(   s    "    N)__name__
__module____qualname__r    r   r   r   r   '   s#        # # # # #r   r   c                      e Zd ZdS )_ResumeIterationN)r   r   r   r   r   r   r   r   ,   s        Dr   r   c                  ,    e Zd ZdZdZed             ZdS )_DatasetKindr   r   c                    | t           j        k    rt          ||||          S | t           j        k    rt	          ||||          S t          d|            )Nzunknown Dataset kind )r!   MAPr   ITERr   NotImplementedError)kinddatasetauto_collate_batch
collate_fn	drop_lasts        r   create_fetcherz_DatasetKind.create_fetcher4   ss     <###%+Z   \&&&*+Z   &&Dd&D&DEEEr   N)r   r   r   r#   r$   staticmethodr+   r   r   r   r!   r!   0   s<        
CDF F \F F Fr   r!   c                      e Zd Zd Zd ZdS )ParentWatchDogc                D    t          j                    | _        d| _        d S NT)osgetppid_parent_pid_parent_aliver   s    r   r   zParentWatchDog.__init__E   s    :<<!r   c                `    | j         r!t          j                    | j        k    | _         | j         S r   )r4   r1   r2   r3   r5   s    r   is_alivezParentWatchDog.is_aliveI   s-     	B!#1A!AD!!r   N)r   r   r   r   r7   r   r   r   r.   r.   D   s2        " " "" " " " "r   r.   return
WorkerInfoc                     t           S )a!  
    Get DataLoader worker process information function, this function is
    used to split data copy in worker process for IterableDataset
    (see :code:`paddle.io.IterableDataset`), worker information contains
    following fields:

    :attr:`num_workers`: total worker process number, see `paddle.io.DataLoader`

    :attr:`id`: the worker process id, count from 0 to :attr:`num_workers - 1`

    :attr:`dataset`: the dataset object in this worker process

    Returns:
        WorkerInfo: an instance of WorkerInfo which contains fields above.

    Notes:
        For more usage and examples, please see :code:`paddle.io.IterableDataset`

    Example:

        .. code-block:: python

            >>> import math
            >>> import paddle
            >>> import numpy as np
            >>> from paddle.io import IterableDataset, DataLoader, get_worker_info

            >>> class SplitedIterableDataset(IterableDataset): # type: ignore[type-arg]
            ...     def __init__(self, start, end):
            ...         self.start = start
            ...         self.end = end
            ...
            ...     def __iter__(self):
            ...         worker_info = get_worker_info()
            ...         if worker_info is None:
            ...             iter_start = self.start
            ...             iter_end = self.end
            ...         else:
            ...             per_worker = int(
            ...                 math.ceil((self.end - self.start) / float(
            ...                     worker_info.num_workers)))
            ...             worker_id = worker_info.id
            ...             iter_start = self.start + worker_id * per_worker
            ...             iter_end = min(iter_start + per_worker, self.end)
            ...
            ...         for i in range(iter_start, iter_end):
            ...             yield np.array([i])
            ...
            >>> place = paddle.CPUPlace()
            >>> dataset = SplitedIterableDataset(start=2, end=9)
            >>> dataloader = DataLoader(
            ...     dataset,
            ...     places=place,
            ...     num_workers=2,
            ...     batch_size=1,
            ...     drop_last=True)
            ...
            >>> for data in dataloader:
            ...     print(data) # doctest: +SKIP("The output depends on the environment.")
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[2]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[6]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[3]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[7]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[4]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[8]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[5]])

    )_worker_infor   r   r   get_worker_infor<   T   s    X r   c                  R     e Zd ZU ded<   ded<   ded<   ded<   dZd Z fd	Z xZS )
r9   intnum_workersidzDataset[Any]r'   seedFc                j    |                                 D ]\  }}t          | ||           d| _        d S r0   )itemssetattr_WorkerInfo__initialized)r   kwargskvs       r   r   zWorkerInfo.__init__   s@    LLNN 	  	 DAqD!Q!r   c                    | j         rt          d| j        j         d          t	                                          ||          S )NzCannot assign attributes to z objects)rE   RuntimeError	__class__r   super__setattr__)r   keyvalrK   s      r   rM   zWorkerInfo.__setattr__   sO     	Pt~/FPPP   ww""3,,,r   )r   r   r   __annotations__rE   r   rM   __classcell__)rK   s   @r   r9   r9      sx         GGGIIIM" " "
- - - - - - - - -r   c                      e Zd ZddZd ZdS )_WorkerExceptionNc                    || _         |pt          j                    }|d         | _        d                    t          j        |           | _        d S )Nr    )r   sysexc_infoexc_typejoin	tracebackformat_exceptionexc_msg)r   r   rW   s      r   r   z_WorkerException.__init__   sC    "-s|~~ wwy98DEEr   c                    d| j          d| j        j         d| j         }t	          | j        dd           r|                     |          |                     |          )NzDataLoader worker(z	) caught z with message:
message)r^   )r   rX   r   r\   getattr)r   msgs     r   reraisez_WorkerException.reraise   sf    r4>rrDM<Rrrdhdprr4=)T22 	----,,,mmC   r   r   )r   r   r   r   ra   r   r   r   rS   rS      s;        F F F F! ! ! ! !r   rS   iװCl   u=& l   y iXl   y isI   l    c           	        t           t          }fdd }|| t          z  | dz	  dg}fd|D             }t          t	          |                    D ]K}t          t	          |                    D ],}||k    r$ |||          ||                             ||<   -Lg }|D ]X}	|	|z  t          z  }
|t
          z  t          z  }|
|z  t          z  }
|
|
t          z	  z  t          z  }
|                    |
           Y|S )Nc                    | z  t           z  } t          z  t           z  | z  t           z  } | | t          z	  z  t           z  } | S r   )MASK32MULT_AXSHIFT)valuehash_const_As    r   hashz_generate_states.<locals>.hash   sH    %/$v-7%/%6/*f4r   c                    t           | z  t          z  }t          |z  t          z  }||z
  t          z  }||t          z	  z  t          z  }|S r   )
MIX_MULT_Lre   
MIX_MULT_Rrg   )xyresult_xresult_yresults        r   mixz_generate_states.<locals>.mix   sF    Nf,Nf,X%/Ff,-7r       r   c                &    g | ]} |          S r   r   ).0entropyrj   s     r   
<listcomp>z$_generate_states.<locals>.<listcomp>  s!    333gDDMM333r   )INIT_AINIT_Bre   rangelenMULT_Brg   append)	base_seedr   hash_const_Brs   	entropiespoolijstatespstaterj   ri   s              @@r   _generate_statesr      sH   LL       I.	RCI3333333D 3t99 6 6s4yy!! 	6 	6AAvv#d1gttDG}}55Q	6 F  \!V+$v-7%/%6/*f4eMr   c                   	 t          j        t                     t          j                     t          j        |           	 dd l}dd l||	z   }|                    |           t          j        |           j                            t          ||	                     n# t          $ r Y nw xY wt          |	|
| |          ad }	 | ||	           t                              || |||          }n#  t!          |	          }Y nxY wd}t#                      }|                                rJ	 |                    t(                    }n# t*          j        $ r Y Bw xY wt/          |t0                    r9|                    |d d f           d}t                              || ||d          }|"|                                s|s
J d            n|                                s|r|\  }}	 ||}d }nct          j        j                            t          j                              5  |                    |          }d d d            n# 1 swxY w Y   t/          |t                     r|                    ||d f           tA          |          \  }}|r+d fd|D             }|                    |||f           n|                    |||f           n# tB          $ ry}t/          |tD                    r5|t          j#        k    r%|                    tI          |	                     d}n%|                    |t!          |	          d f           Y d }~nd }~ww xY w|                                Jn# tJ          $ r Y n  xY w|rt                       n# |rt                       w w xY w|                                r*|&                                 |'                                 d S d S )	Nr   )r@   r?   r'   rA   FTz#get None when worker done_event set)placec                z    t          j                    }|                    | t          j                               |S r   )r   DenseTensorsetCPUPlace)arr	lodtensors     r   numpy2lodtensorz%_worker_loop.<locals>.numpy2lodtensor  s0    $($4$6$6	!c4=??;;;((r   c                x    g | ]6}t          |j                  r |          n|                                7S r   )
isinstancendarray
get_tensor)rv   bnpr   s     r   rx   z _worker_loop.<locals>.<listcomp>  sV     # # #   *!RZ880OOA...!"	# # #r   )(r
   registerr   r   _set_process_signal_handler(_set_max_memory_map_allocation_pool_sizerandomnumpyrA   paddler   ImportErrorr9   r;   r!   r+   rS   r.   r7   getr	   queueEmptyr   r   putis_setbasedygraphguardr   fetchr   	ExceptionStopIterationr$   r   KeyboardInterruptcancel_join_threadclose)r'   dataset_kindindices_queue	out_queue
done_eventr(   r)   r*   init_fnr   r?   use_shared_memoryr   shm_cache_sizer   rA   init_exceptionfetcheriterator_drainedparent_watch_dogdataidxindicesbatch	structuretensor_lister   r   s                              @@r   _worker_loopr     s    z
 	%m444 	(***5nEEE
	CMMM y(DKKKINN+IyAABBBB  	 	 	D	 "#	
 
 
 	9"	""""11g'9:y GG	9-i88NNN )++'')) D	;$(()ABB;    $ 011 tT40111#( &55 '+=z4   |!((** .>  9 >    "" &6 LC*;!-*E%)NN  ,229J9J2KK 7 7 'g 6 67 7 7 7 7 7 7 7 7 7 7 7 7 7 7 e%566 6MM3t"4555#1%#8#8 y$ ;) ) )
# # # # # "'# # #K MM3Y"?@@@@MM3y"9::::=  L L Lq-00L$(999MM"?	"J"JKKK'+$$MM3(8(C(CT"JKKKLM '')) D	;J     	OOO  	OOOO	 $$&&& s   A M< B$ AM< $
B1.M< 0B11M< 
+C6 5M< 6D	*M< 2E M< EM< EBM< 1=K  .IK  IK  IK  BM<  
M#*A/MM< M##M< ;N <
N
N N

N N1)r8   r9   )r   r   )r   ).
__future__r   r1   r   rV   rZ   typingr   r   r   r   r   	frameworkr   multiprocess_utilsr	   r
   r   r   r   r   flatr   	paddle.ior   r   r   r!   r.   r;   r<   r9   rS   ry   rf   rz   r}   rl   rm   dtypeuint32itemsizerg   re   r   r   r   r   r   <module>r      s   # " " " " " 				  



     % % % % % % % %                    
 A @ @ @ @ @ @ @             "!!!!!!# # # # # # # #
	 	 	 	 	 	 	 	F F F F F F F F(" " " " " " " " L L L L^- - - - - - - -*! ! ! ! ! ! ! !\ 
			

	")			%	)Q	.	& & & &n M M M M M Mr   