{-j d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl Z
d dl m Z d dl
mZ erd dlmZ d dlmZ ed Zg ZdZd Z G d
d e ZdS ) ) annotationsN)
TYPE_CHECKINGLiteral)_check_exists_and_download)Dataset)Patterntraintestz6https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz 7c2ac02c03563afcf9b574c7e56c153ac e Zd ZU dZded< ded< ded< ded < ded
< d d!dZd"dZd#dZd$dZd%d Z d&d Z
d S )'Imdba
Implementation of `IMDB `_ dataset.
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None.
mode(str): 'train' 'test' mode. Default 'train'.
cutoff(int): cutoff number for building word dictionary. Default 150.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True.
Returns:
Dataset: instance of IMDB dataset
Examples:
.. code-block:: pycon
>>> # doctest: +TIMEOUT(75)
>>> import paddle
>>> from paddle.text.datasets import Imdb
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, doc, label):
... return paddle.sum(doc), label
>>> imdb = Imdb(mode='train')
>>> for i in range(10):
... doc, label = imdb[i]
... doc = paddle.to_tensor(doc)
... label = paddle.to_tensor(label)
...
... model = SimpleNet()
... image, label = model(doc, label)
... print(doc.shape, label.shape)
paddle.Size([121]) paddle.Size([1])
paddle.Size([115]) paddle.Size([1])
paddle.Size([386]) paddle.Size([1])
paddle.Size([471]) paddle.Size([1])
paddle.Size([585]) paddle.Size([1])
paddle.Size([206]) paddle.Size([1])
paddle.Size([221]) paddle.Size([1])
paddle.Size([324]) paddle.Size([1])
paddle.Size([166]) paddle.Size([1])
paddle.Size([598]) paddle.Size([1])
str | None data_file_ImdbDataSetModemodedict[str, int]word_idxlistdocslabelsNr
TcutoffintdownloadboolreturnNonec R | dv s
J d| | | _ || _ | j .|s
J d t |t t
d| | _ | | | _ | d S )Nr z(mode should be 'train', 'test', but got z>data_file is not set and downloading automatically is disabledimdb) lowerr r r URLMD5_build_work_dictr
_load_anno)selfr r r r s Y/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/text/datasets/imdb.py__init__z
Imdb.__init__b s zz||
=d < <
JJLL " > !
P
8 83VX DN
--f55
c t j t }t j d }| | D ]}|D ]}||xx dz
cc< fd| D }t |d }t t | \ }}t t t |t t | } t | | d< | S )Nz/aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$ c , g | ]}|d k |S )r+ ).0xr s r'
z)Imdb._build_work_dict.. s" CCC1QqTF]]Q]]]r) c $ | d | d fS )Nr+ r r- )r/ s r' z'Imdb._build_work_dict.. s qteQqT] r) )key)
collections defaultdictr recompile _tokenizeitemssortedr zipdictrangelen)
r& r word_freqpatterndocword
dictionarywords_r s
` r' r$ zImdb._build_work_dict~ s +C00 * OPP>>'** % %C
%
%$1$
% DCCC 1 1CCC I+B+BCCC
Z())qSc%jj(9(9 : :;;<< JJr) rA Pattern[str]list[list[str]]c V g }t j | j 5 }| }|t | |j r| | |
d d t j
d | }|d d d n # 1 swxY w Y |S )Ns
zlatin-1)tarfileopenr nextr matchnameappend extractfilereadrstrip translatestring punctuationencoder! split)r& rA datatarftfs r' r9 zImdb._tokenize s%
\$.
)
)
!TB.
bg..// KK ((,, "4);)B)B9)M)MNN
YY[[ .
!
!
!
!
!
!
!
!
!
!
!
!
!
!
! s C6D D"%D"c t j d j d }t j d j d } j d g _ g _ | D ]C} j fd|D j d D | D ]C} j fd|D j d Dd S ) NzaclImdb/z
/pos/.*\.txt$z
/neg/.*\.txt$r4 c F g | ] }j | S r- r getr. wUNKr& s r' r0 z#Imdb._load_anno.. + E E EAdm / /3 7 7 E E Er) r c F g | ] }j | S r- r] r_ s r' r0 z#Imdb._load_anno.. rb r) r+ )r7 r8 r r r r r9 rO )r& pos_pattern neg_patternrB ra s ` @r' r% zImdb._load_anno s& j!ETY!E!E!EFF j!ETY!E!E!EFF mG$ >>+.. " "C I E E E E E E E E F F F K q ! ! ! !>>+.. " "C I E E E E E E E E F F F K q ! ! ! ! " "r) idx1tuple[npt.NDArray[np.int_], npt.NDArray[np.int_]]c t j | j | t j | j | g fS N)nparrayr r )r& rf s r' __getitem__zImdb.__getitem__ s3 3 (("(DK4D3E*F*FGGr) c * t | j S ri )r? r )r& s r' __len__z Imdb.__len__ s 49~~ r) )Nr
r T)
r r r r r r r r r r )r r r r )rA rG r rH )r r )rf r r rg )r r ) __name__
__module__ __qualname____doc____annotations__r( r$ r9 r% rl rn r- r) r' r r ' s 2 2h JJJLLL !%!( 8 &
"
"
"
" H H H H
r) r )
__future__r r5 r7 rT rJ typingr r numpyrj paddle.dataset.commonr paddle.ior r numpy.typingnptr __all__r" r# r r- r) r' r| s # " " " " "
) ) ) ) ) ) ) ) < < < < < < 0 /
>(O O O O O 7 O O O O O r)