
    i                        S SK r S SKrS SKrS SKJr  S SKrS SKr " S S\	5      r
 " S S\
5      r " S S\5      r " S	 S
\
5      r " S S\
5      r " S S\
5      r " S S\
5      r " S S\
5      r " S S\
5      r " S S\
5      r " S S\5      r " S S\
5      r " S S\
5      r " S S\5      r " S S \5      r " S! S"\5      r " S# S$\
5      r " S% S&\
5      r " S' S(\5      r " S) S*\	5      r " S+ S,\	5      rg)-    N)
functionalc                   J    \ rS rSrSrSS jrS rS rS r   SS jr	S	 r
S
rg)BaseRecLabelDecode   )Convert between text-label and text-indexNc                    SU l         SU l        SU l        / U l        Uc  SU l        [	        U R                  5      nO[        US5       nUR                  5       nU HM  nUR                  S5      R                  S5      R                  S5      nU R                  R                  U5        MO     S S S 5        U(       a  U R                  R                  S	5        [	        U R                  5      nS
U;   a  SU l        U R                  U5      n0 U l        [        U5       H  u  pxXpR                  U'   M     X0l        g ! , (       d  f       N= f)NsoseosF$0123456789abcdefghijklmnopqrstuvwxyzrbutf-8
z
 arabicT)beg_strend_strreversecharacter_strlistopen	readlinesdecodestripappendadd_special_chardict	enumerate	character)	selfcharacter_dict_pathuse_space_chardict_characterfinlineslineichars	            k/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddleocr/ppocr/postprocess/rec_postprocess.py__init__BaseRecLabelDecode.__init__   s   &!GD!$"4"45N)40C!D;;w/55d;AA&ID&&--d3 " 1
 ""))#.!$"4"45N..#..~>	 0GAIIdO 1' 10s   	A$D::
Ec                 "   / nSnU HX  n[        [        R                  " SU5      5      (       d,  US:w  a  UR                  U5        UR                  U5        SnMT  X4-  nMZ     US:w  a  UR                  U5        SR	                  US S S2   5      $ )N z[a-zA-Z0-9 :*./%+-])boolresearchr   join)r   predpred_re	c_currentcs        r(   pred_reverseBaseRecLabelDecode.pred_reverse5   s    	A		"7;<<?NN9-q!		  ?NN9%wwwtt}%%    c                     U$ N r   r"   s     r(   r   #BaseRecLabelDecode.add_special_charE   s    r8   c           	      6   Sn/ n/ n/ n/ n/ n[         R                  " US:H  5      S   n	[        U5       GH  u  pSUs=::  a  S::  a  O  OSnO*[        [        R
                  " SU5      5      (       a  SnOS	nUS
:X  aD  US:X  a>  U
S-   [        U5      :  a,  [        [        R
                  " SXS-      5      5      (       a  SnUS:X  a  US:X  a  SnUS:X  a  UnX<:w  aH  [        U5      S:w  a7  UR                  U5        UR                  U5        UR                  U5        / n/ nUnUS	:w  d  M  UR                  U5        UR                  X   5        GM     [        U5      S:w  a3  UR                  U5        UR                  U5        UR                  U5        XgU4$ )u8  
Group the decoded characters and record the corresponding decoded positions.

Args:
    text: the decoded text
    selection: the bool array that identifies which columns of features are decoded as non-separated characters
Returns:
    word_list: list of the grouped words
    word_col_list: list of decoding positions corresponding to each character in the grouped word
    state_list: list of marker to identify the type of grouping words, including two types of grouping words:
                - 'cn': continuous chinese characters (e.g., 你好啊)
                - 'en&num': continuous english characters (e.g., hello), number (e.g., 123, 1.123), or mixed of them connected by '-' (e.g., VGG-16)
                The remaining characters in text are treated as separators between groups (e.g., space, '(', ')', etc.).
NTr   u   一u   鿿cnz[a-zA-Z0-9]zen&numsplitter.   z[0-9]-)npwherer   r.   r/   r0   lenr   )r   text	selectionstateword_contentword_col_content	word_listword_col_list
state_list	valid_colc_ir'   c_states                r(   get_word_info BaseRecLabelDecode.get_word_infoH   s    	
HHY$./2	"4IC4+8+biit455"$ X%!Gc$i'7DqM:;;" 1"}|$)$$\2!(()9:%%e,#%L')$
"##D) ''	7E )H |!\*  !12e$33r8   c           	      H   / nU R                  5       n[        U5      n[        U5       GHo  n[        R                  " [        X   5      [
        S9n	U(       a  X   SS X   SS :g  U	SS& U H  n
XU   U
:g  -  n	M     X   U	    Vs/ s H  oR                  U   PM     nnUb  X(   U	   nOS/[        U	5      -  n[        U5      S:X  a  S/nSR                  U5      nU R                  (       a  U R                  U5      nU(       a]  U R                  X5      u  nnnUR                  U[        R                  " U5      R                  5       [        X   5      UUU/45        GM:  UR                  U[        R                  " U5      R                  5       45        GMr     U$ s  snf #convert text-index into text-label.dtyperB   Nr-   r   r,   )get_ignored_tokensrF   rangerD   onesr.   r   r1   r   r6   rR   r   meantolist)r   
text_index	text_probis_remove_duplicatereturn_word_boxresult_listignored_tokens
batch_size	batch_idxrH   ignored_tokentext_id	char_list	conf_listrG   rL   rM   rN   s                     r(   r   BaseRecLabelDecode.decode   s    002_
z*IJ$9 :$GI" * 5ab 9Z=RSVTV=W W	!"!/	2mCC	 "0 8B7LY7W7WGw'7W   $%0;	C#i.0	9~"C	779%D||((.7;7I7I84	=* ""	*113
 56%)&		 ""D"'')*<*C*C*E#FGO +P Cs   Fc                     S/$ Nr   r;   r   s    r(   rY   %BaseRecLabelDecode.get_ignored_tokens   s	    s
r8   )r   r   r   r   r   r   NF)NFF)__name__
__module____qualname____firstlineno____doc__r)   r6   r   rR   r   rY   __static_attributes__r;   r8   r(   r   r      s1    3(6& @4J !3jr8   r   c                   @   ^  \ rS rSrSrSU 4S jjrSS jrS rSrU =r	$ )CTCLabelDecode   r   c                 ,   > [         [        U ]  X5        g r:   )superrw   r)   r   r    r!   kwargs	__class__s       r(   r)   CTCLabelDecode.__init__       nd,-@Qr8   c                    [        U[        5      (       d  [        U[        5      (       a  US   n[        U[        R                  5      (       a  UR                  5       nUR                  SS9nUR                  SS9nU R                  UUSUS9nU(       a4  [        U5       H%  u  pUS   U	   nUS   nU
S   S   X-  -  U
S   S'   M'     Uc  U$ U R                  U5      nX4$ )	Nr-      axisT)r`   ra   wh_ratio_listmax_wh_ratior   )

isinstancetupler   paddleTensornumpyargmaxmaxr   r   )r   predslabelra   argsr|   	preds_idx
preds_probrG   rec_idxrecwh_ratior   s                r(   __call__CTCLabelDecode.__call__   s    eU##z%'>'>"IEeV]]++KKMELLaL(	YYAY&
{{ $+	  
  )$!/27;%n5F1I)@AAq	 !0 =KE"{r8   c                     S/U-   nU$ )Nblankr;   r<   s     r(   r   CTCLabelDecode.add_special_char   s    !^3r8   r;   ro   
rp   rq   rr   rs   rt   r)   r   r   ru   __classcell__r}   s   @r(   rw   rw      s    3R. r8   rw   c                   L   ^  \ rS rSrSrSSS/SS4U 4S jjrS	U 4S jjrSrU =r$ )
DistillationCTCLabelDecode   3
Convert
Convert between text-label and text-index
NFstudentc                    > [         [        U ]  X5        [        U[        5      (       d  U/nX0l        X@l        XPl        g r:   )rz   r   r)   r   r   
model_namekey
multi_headr   r    r!   r   r   r   r|   r}   s          r(   r)   #DistillationCTCLabelDecode.__init__   =     	($8	
 *d++$J$$r8   c                   > [        5       nU R                   He  nX   nU R                  b  XpR                     nU R                  (       a  [	        U[         5      (       a  US   n[
        TU ]  " U/UQ7SU0UD6XV'   Mg     U$ )Nctcr   r   r   r   r   r   rz   r   	r   r   r   r   r|   outputnamer2   r}   s	           r(   r   #DistillationCTCLabelDecode.__call__   w    OOD;Dxx#HH~:dD#9#9E{ 7+DOOOOFL $ r8   r   r   r   r:   	rp   rq   rr   rs   rt   r)   r   ru   r   r   s   @r(   r   r      *     !;%&	 	r8   r   c                   V   ^  \ rS rSrSrS
U 4S jjrS rS
S jrSS jrS r	S r
S	rU =r$ )AttnLabelDecodei
  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r   r)   r{   s       r(   r)   AttnLabelDecode.__init__      ot-.ARr8   c                 b    SU l         SU l        UnU R                   /U-   U R                  /-   nU$ Nr	   r
   r   r   r<   s     r(   r    AttnLabelDecode.add_special_char  6    ',,.8DLL>Ir8   c           	         / nU R                  5       nU R                  5       u  pg[        U5      n[        U5       GH  n	/ n
/ n[        [        X   5      5       H  nX   U   U;   a  M  [        X   U   5      [        U5      :X  a    O~U(       a  US:  a  X   US-
     X   U   :X  a  MQ  U
R	                  U R
                  [        X   U   5         5        Ub  UR	                  X)   U   5        M  UR	                  S5        M     SR                  U
5      nUR	                  U[        R                  " U5      R                  5       45        GM     U$ rV   r   rB   r,   
rY   rF   rZ   intr   r   r1   rD   r\   r]   r   r^   r_   r`   rb   rc   beg_idxend_idxrd   re   rh   ri   idxrG   s                 r(   r   AttnLabelDecode.decode  E   002!446_
z*IIIS!678(-?z,S12c'lB& a&1#':j>STW>XX   J4I#4N0O!PQ($$Y%9#%>?$$Q'! 9" 779%Dbggi&8&?&?&ABC+ +, r8   c                     [        U[        R                  5      (       a  UR                  5       nUR	                  SS9nUR                  SS9nU R                  XVSS9nUc  U$ U R                  USS9nXr4$ )
text = self.decode(text)
if label is None:
    return text
else:
    label = self.decode(label, is_remove_duplicate=False)
    return text, label
r   r   Fr`   r   r   r   r   r   r   r   r   r   r   r   r|   r   r   rG   s           r(   r   AttnLabelDecode.__call__5  sv     eV]]++KKMELLaL(	YYAY&
{{9e{L=KEu={r8   c                 L    U R                  S5      nU R                  S5      nX/$ Nbegendget_beg_end_flag_idxr   r   r   s      r(   rY   "AttnLabelDecode.get_ignored_tokensI  +    ++E2++E2!!r8   c                     US:X  a/  [         R                  " U R                  U R                     5      nU$ US:X  a/  [         R                  " U R                  U R                     5      nU$  SU-  5       eNr   r   +unsupported type %s in get_beg_end_flag_idxrD   arrayr   r   r   r   
beg_or_endr   s      r(   r   $AttnLabelDecode.get_beg_end_flag_idxN  k    ((499T\\23C
 
	 5 ((499T\\23C 
 UG*TT5r8   r   ro   r:   rp   rq   rr   rs   rt   r)   r   r   r   rY   r   ru   r   r   s   @r(   r   r   
  s*    3S<("
 r8   r   c                   V   ^  \ rS rSrSrS
U 4S jjrS rS
S jrSS jrS r	S r
S	rU =r$ )RFLLabelDecodeiX  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r   r)   r{   s       r(   r)   RFLLabelDecode.__init__[  r   r8   c                 b    SU l         SU l        UnU R                   /U-   U R                  /-   nU$ r   r   r<   s     r(   r   RFLLabelDecode.add_special_char^  r   r8   c           	         / nU R                  5       nU R                  5       u  pg[        U5      n[        U5       GH  n	/ n
/ n[        [        X   5      5       H  nX   U   U;   a  M  [        X   U   5      [        U5      :X  a    O~U(       a  US:  a  X   US-
     X   U   :X  a  MQ  U
R	                  U R
                  [        X   U   5         5        Ub  UR	                  X)   U   5        M  UR	                  S5        M     SR                  U
5      nUR	                  U[        R                  " U5      R                  5       45        GM     U$ r   r   r   s                 r(   r   RFLLabelDecode.decodee  r   r8   c                    [        U[        5      (       d  [        U[        5      (       ay  Uu  pV[        U[        R                  5      (       a  UR                  5       nUR                  SS9nUR                  SS9nU R                  XxSS9n	Uc  U	$ U R                  USS9nX4$ Un[        U[        R                  5      (       a  UR                  5       n/ n
U H3  n[        [        R                  " U5      5      nU
R                  U5        M5     Uc  U
$ U R                  USS9nU Vs/ s H  n[        US   5      PM     nnX4$ s  snf )Nr   r   Fr   r   )r   r   r   r   r   r   r   r   r   roundrD   sumr   rF   )r   r   r   r   r|   cnt_outputsseq_outputsr   r   rG   
cnt_lengthlenslengthress                 r(   r   RFLLabelDecode.__call__  s9   eU##z%'>'>',$K+v}}55)//1#***2I$a0J;;y%;PD}KK5KAE;  K+v}}55)//1J#rvvd|,!!&) $ }!!KK5KAE-23Ucc#a&kUF3%% 4s   )Ec                 L    U R                  S5      nU R                  S5      nX/$ r   r   r   s      r(   rY   !RFLLabelDecode.get_ignored_tokens  r   r8   c                     US:X  a/  [         R                  " U R                  U R                     5      nU$ US:X  a/  [         R                  " U R                  U R                     5      nU$  SU-  5       er   r   r   s      r(   r   #RFLLabelDecode.get_beg_end_flag_idx  r   r8   r   ro   r:   r   r   s   @r(   r   r   X  s*    3R<&:"
 r8   r   c                   V   ^  \ rS rSrSrS
U 4S jjrS rS rS rS
S jr	SS jr
S	rU =r$ )SEEDLabelDecodei  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r   r)   r{   s       r(   r)   SEEDLabelDecode.__init__  r   r8   c                 z    SU l         SU l        SU l        XR                  U R                   U R                  /-   nU$ )Npaddingr
   unknown)padding_strr   r   r<   s     r(   r    SEEDLabelDecode.add_special_char  s;    $ '<<9I9I4<<*XXr8   c                 *    U R                  S5      nU/$ )Nr
   r   )r   r   s     r(   rY   "SEEDLabelDecode.get_ignored_tokens  s    ++E2yr8   c                     US:X  a/  [         R                  " U R                  U R                     5      nU$ US:X  a/  [         R                  " U R                  U R                     5      nU$  SU-  5       e)Nr	   r
   r   r   r   s      r(   r   $SEEDLabelDecode.get_beg_end_flag_idx  r   r8   c           	      l   / nU R                  5       u  n[        U5      n[        U5       GH  n/ n/ n	[        [        X   5      5       H  n
[        X   U
   5      [        U5      :X  a    O~U(       a  U
S:  a  X   U
S-
     X   U
   :X  a  MD  UR	                  U R
                  [        X   U
   5         5        Ub  U	R	                  X'   U
   5        M  U	R	                  S5        M     SR                  U5      nUR	                  U[        R                  " U	5      R                  5       45        GM     U$ r   r   )r   r^   r_   r`   rb   r   rd   re   rh   ri   r   rG   s               r(   r   SEEDLabelDecode.decode  s#   ++-	_
z*IIIS!678z,S12c'lB& a&1#':j>STW>XX   J4I#4N0O!PQ($$Y%9#%>?$$Q' 9 779%Dbggi&8&?&?&ABC' +( r8   c                 $   US   n[        U[        R                  5      (       a  UR                  5       nSU;   a  US   nUS   nO$US   R	                  SS9nUS   R                  SS9nU R                  XVSS9nUc  U$ U R                  USS9nXr4$ )r   rec_predrec_pred_scoresr   r   Fr   r   r   s           r(   r   SEEDLabelDecode.__call__  s     *%	i//!)I%j)I01Jj)00a08Iz*..A.6J{{9e{L=KEu={r8   )r   r   r   ro   r:   )rp   rq   rr   rs   rt   r)   r   rY   r   r   r   ru   r   r   s   @r(   r   r     s*    3S6 r8   r   c                   V   ^  \ rS rSrSrS
U 4S jjrSS jrS
S jrS rS r	S r
S	rU =r$ )SRNLabelDecodei  r   c                 Z   > [         [        U ]  X5        UR                  SS5      U l        g Nmax_text_length   )rz   r  r)   getr  r{   s       r(   r)   SRNLabelDecode.__init__   s'    nd,-@Q%zz*;R@r8   c                     US   n[        U R                  5      S-   n[        U[        R                  5      (       a  UR                  5       n[        R                  " USU/5      n[        R                  " USS9n[        R                  " USS9n[        R                  " USU R                  /5      n[        R                  " USU R                  /5      nU R                  Xx5      n	Uc  U R                  XxSS9n	U	$ U R                  U5      nX4$ )Npredictr   r-   rB   r   Fr   )rF   r   r   r   r   r   rD   reshaper   r   r  r   )
r   r   r   r   r|   r2   char_numr   r   rG   s
             r(   r   SRNLabelDecode.__call__  s    Yt))*Q.dFMM**::<Dzz$X/IId+	VVDq)
JJy2t/C/C*DE	ZZ
R1E1E,FG
{{91=;;y%;PDKE"{r8   c           	      @   / nU R                  5       n[        U5      n[        U5       H  n/ n/ n	[        [        X   5      5       H  n
X   U
   U;   a  M  U(       a  U
S:  a  X   U
S-
     X   U
   :X  a  M2  UR                  U R                  [        X   U
   5         5        Ub  U	R                  X'   U
   5        My  U	R                  S5        M     SR                  U5      nUR                  U[        R                  " U	5      R                  5       45        M     U$ r   )
rY   rF   rZ   r   r   r   r1   rD   r\   r]   )r   r^   r_   r`   rb   rc   rd   re   rh   ri   r   rG   s               r(   r   SRNLabelDecode.decode  s   002_
z*IIIS!678(-?& a&1#':j>STW>XX   J4I#4N0O!PQ($$Y%9#%>?$$Q' 9  779%Dbggi&8&?&?&ABC) +* r8   c                 :    XR                   U R                  /-   nU$ r:   r   r<   s     r(   r   SRNLabelDecode.add_special_char7  s    '<<*FFr8   c                 L    U R                  S5      nU R                  S5      nX/$ r   r   r   s      r(   rY   !SRNLabelDecode.get_ignored_tokens;  r   r8   c                     US:X  a/  [         R                  " U R                  U R                     5      nU$ US:X  a/  [         R                  " U R                  U R                     5      nU$  SU-  5       er   r   r   s      r(   r   #SRNLabelDecode.get_beg_end_flag_idx@  r   r8   r  ro   r:   )rp   rq   rr   rs   rt   r)   r   r   r   rY   r   ru   r   r   s   @r(   r  r    s*    3A,:"
 r8   r  c                   f   ^  \ rS rSrSrSrSrSrSU 4S jjrSS jr	SS jr
S	 rSS
 jrS rSrU =r$ )ParseQLabelDecodeiJ  r   z[B]z[E]z[P]c                 Z   > [         [        U ]  X5        UR                  SS5      U l        g r  )rz   r!  r)   r  r  r{   s       r(   r)   ParseQLabelDecode.__init__Q  s(    /0CT%zz*;R@r8   c                 H   [        U[        5      (       a  US   nOUn[        U R                  5      S-   n[        U[        R
                  5      (       a  UR                  5       nUR                  S S u  px[        R                  " USU/5      n[        R                  " USS9n	[        R                  " USS9n
[        R                  " XU/5      n	[        R                  " XU/5      n
Uc  U R                  XSS9nU$ U R                  XSS9nU R                  US S5      nX4$ )Nr  rB   r   r-   r   F)raw)r   r   rF   r   r   r   r   shaperD   r  r   r   r   )r   r   r   r   r|   r2   r  BLr   r   rG   s               r(   r   ParseQLabelDecode.__call__U  s   eT""#DD ""#a' 	 dFMM**::<Dzz"1~zz$X/IId+	VVDq)
JJya&1	ZZ
F3
=;;y%;@DK{{9e{<E4/{r8   c           	      D   / nU R                  5       n[        U5      n[        U5       H  n/ n/ n	XSS24   n
SnUb  X'SS24   nU(       d  U R                  X5      u  p[        [        U
5      5       H_  nX   U;   a  M  UR	                  U R
                  [        X   5         5        Ub  U	R	                  X   5        MN  U	R	                  S5        Ma     SR                  U5      nUR	                  U[        R                  " U	5      R                  5       45        M     U$ )rV   NrB   r,   )rY   rF   rZ   _filterr   r   r   r1   rD   r\   r]   )r   r^   r_   r%  rb   rc   rd   re   rh   ri   indexprobr   rG   s                 r(   r   ParseQLabelDecode.decoder  s   002_
z*III!|,ED$ A."ll57SZ(:/  EJ!@A($$TY/$$Q' ) 779%Dbggi&8&?&?&ABC- +0 r8   c                 X    U R                   /U-   U R                  U R                  /-   nU$ r:   )EOSBOSPADr<   s     r(   r   "ParseQLabelDecode.add_special_char  s*    ((n4$((7KKr8   c                     UR                  5       n UR                  U R                  U R                     5      nUS U nUb  US US-    nX4$ ! [         a    [        U5      n N*f = f)NrB   )r]   r,  r   r0  
ValueErrorrF   )r   idsprobseos_idxs       r(   r+  ParseQLabelDecode._filter  sr    jjl	ii		$(( 34G (7m-GaK(Ez  	#hG	s   (A A%$A%c                     U R                   U R                     U R                   U R                     U R                   U R                     /$ r:   )r   r1  r0  r2  rm   s    r(   rY   $ParseQLabelDecode.get_ignored_tokens  s6    		$((#TYYtxx%8$))DHH:MNNr8   r  ro   r:   )rp   rq   rr   rs   rt   r1  r0  r2  r)   r   r   r   r+  rY   ru   r   r   s   @r(   r!  r!  J  s<    3
C
C
CA:@
O Or8   r!  c                   P   ^  \ rS rSrSrS	U 4S jjrS rS	S jrS
S jrS r	Sr
U =r$ )SARLabelDecodei  r   c                 Z   > [         [        U ]  X5        UR                  SS5      U l        g N	rm_symbolF)rz   r=  r)   r  r@  r{   s       r(   r)   SARLabelDecode.__init__  s%    nd,-@QK7r8   c                     SnSnSnX/-   n[        U5      S-
  U l        X/-   n[        U5      S-
  U l        [        U5      S-
  U l        X/-   n[        U5      S-
  U l        U$ Nz	<BOS/EOS>z<UKN><PAD>rB   rF   unknown_idx	start_idxr   padding_idxr   r"   beg_end_strunknown_strr   s        r(   r   SARLabelDecode.add_special_char  {    !'-7~.2'-7^,q0>*Q.'-7~.2r8   c           	      >   / nU R                  5       n[        U5      n[        U5       GHo  n/ n/ n	[        [        X   5      5       H  n
X   U
   U;   a  M  [        X   U
   5      [        U R                  5      :X  a  Uc  U
S:X  a  MB    O~U(       a  U
S:  a  X   U
S-
     X   U
   :X  a  Mf  UR                  U R                  [        X   U
   5         5        Ub  U	R                  X'   U
   5        M  U	R                  S5        M     SR                  U5      nU R                  (       a8  [        R                  " S5      nUR                  5       nUR                  SU5      nUR                  U[        R                  " U	5      R                  5       45        GMr     U$ rV   r   rB   r,   u   [^A-Z^a-z^0-9^一-龥]rY   rF   rZ   r   r   r   r   r1   r@  r/   compilelowersubrD   r\   r]   r   r^   r_   r`   rb   rc   rd   re   rh   ri   r   rG   comps                r(   r   SARLabelDecode.decode  y   002_
z*IIIS!678(-?z,S12c$,,6GG (SAX & a&1#':j>STW>XX   J4I#4N0O!PQ($$Y%9#%>?$$Q'' 9( 779%D~~zz"@Azz|xxD)bggi&8&?&?&ABC9 +: r8   c                     [        U[        R                  5      (       a  UR                  5       nUR	                  SS9nUR                  SS9nU R                  XVSS9nUc  U$ U R                  USS9nXr4$ Nr   r   Fr   r   r   s           r(   r   SARLabelDecode.__call__  t    eV]]++KKMELLaL(	YYAY&
{{9e{L=KEu={r8   c                     U R                   /$ r:   rH  rm   s    r(   rY   !SARLabelDecode.get_ignored_tokens        !!r8   r   rH  r@  rG  rF  ro   r:   rp   rq   rr   rs   rt   r)   r   r   r   rY   ru   r   r   s   @r(   r=  r=    %    38
#J" "r8   r=  c                   P   ^  \ rS rSrSrS	U 4S jjrS rS	S jrS
S jrS r	Sr
U =r$ )SATRNLabelDecodei  r   c                 Z   > [         [        U ]  X5        UR                  SS5      U l        g r?  )rz   rd  r)   r  r@  r{   s       r(   r)   SATRNLabelDecode.__init__  s&    ./BSK7r8   c                     SnSnSnX/-   n[        U5      S-
  U l        X/-   n[        U5      S-
  U l        [        U5      S-
  U l        X/-   n[        U5      S-
  U l        U$ rC  rE  rI  s        r(   r   !SATRNLabelDecode.add_special_char  rM  r8   c           	      >   / nU R                  5       n[        U5      n[        U5       GHo  n/ n/ n	[        [        X   5      5       H  n
X   U
   U;   a  M  [        X   U
   5      [        U R                  5      :X  a  Uc  U
S:X  a  MB    O~U(       a  U
S:  a  X   U
S-
     X   U
   :X  a  Mf  UR                  U R                  [        X   U
   5         5        Ub  U	R                  X'   U
   5        M  U	R                  S5        M     SR                  U5      nU R                  (       a8  [        R                  " S5      nUR                  5       nUR                  SU5      nUR                  U[        R                  " U	5      R                  5       45        GMr     U$ rO  rP  rT  s                r(   r   SATRNLabelDecode.decode  rW  r8   c                     [        U[        R                  5      (       a  UR                  5       nUR	                  SS9nUR                  SS9nU R                  XVSS9nUc  U$ U R                  USS9nXr4$ rY  r   r   s           r(   r   SATRNLabelDecode.__call__+  r[  r8   c                     U R                   /$ r:   r]  rm   s    r(   rY   #SATRNLabelDecode.get_ignored_tokens8  r_  r8   r`  ro   r:   ra  r   s   @r(   rd  rd    rb  r8   rd  c                   L   ^  \ rS rSrSrSSS/SS4U 4S jjrS	U 4S jjrSrU =r$ )
DistillationSARLabelDecodei<  r   NFr   c                    > [         [        U ]  X5        [        U[        5      (       d  U/nX0l        X@l        XPl        g r:   )rz   rp  r)   r   r   r   r   r   r   s          r(   r)   #DistillationSARLabelDecode.__init__B  r   r8   c                   > [        5       nU R                   He  nX   nU R                  b  XpR                     nU R                  (       a  [	        U[         5      (       a  US   n[
        TU ]  " U/UQ7SU0UD6XV'   Mg     U$ )Nsarr   r   r   s	           r(   r   #DistillationSARLabelDecode.__call__U  r   r8   r   r:   r   r   s   @r(   rp  rp  <  r   r8   rp  c                   J   ^  \ rS rSrSrSU 4S jjrS rS	S jrS	S jrSr	U =r
$ )
PRENLabelDecodeia  r   c                 ,   > [         [        U ]  X5        g r:   )rz   rw  r)   r{   s       r(   r)   PRENLabelDecode.__init__d  r   r8   c                 J    SnSnSnX#U/U-   nSU l         SU l        SU l        U$ )NrD  z<EOS>z<UNK>r   rB   r   )rH  r   rF  )r   r"   r   r   rK  s        r(   r    PRENLabelDecode.add_special_charg  s=    %<~Mr8   c           	         / n[        U5      n[        U5       GH   n/ n/ n[        [        X   5      5       H  nX   U   U R                  :X  a    OX   U   U R                  U R                  4;   a  M=  UR                  U R                  [        X   U   5         5        Ub  UR                  X%   U   5        M  UR                  S5        M     SR                  U5      n	[        U	5      S:  a8  UR                  U	[        R                  " U5      R                  5       45        GM  UR                  S5        GM#     U$ )rV   rB   r,   r   )r,   rB   )rF   rZ   r   rH  rF  r   r   r   r1   rD   r\   r]   )
r   r^   r_   rb   rd   re   rh   ri   r   rG   s
             r(   r   PRENLabelDecode.decodes  s!   _
z*IIIS!678(-=(-$2B2BDDTDT1UU  J4I#4N0O!PQ($$Y%9#%>?$$Q' 9 779%D4y1}""D"'')*<*C*C*E#FG ""7+' +( r8   c                     [        U[        R                  5      (       a  UR                  5       nUR	                  SS9nUR                  SS9nU R                  XV5      nUc  U$ U R                  U5      nXr4$ Nr   r   r   r   s           r(   r   PRENLabelDecode.__call__  sl    eV]]++KKMELLaL(	YYAY&
{{91=KE"{r8   )r   rH  rF  ro   r:   )rp   rq   rr   rs   rt   r)   r   r   r   ru   r   r   s   @r(   rw  rw  a  s     3S
6	 	r8   rw  c                   J   ^  \ rS rSrSrSU 4S jjrS	S jrS rS
S jrSr	U =r
$ )NRTRLabelDecodei  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r  r)   r{   s       r(   r)   NRTRLabelDecode.__init__  r   r8   c                    [        U5      S:X  a  US   nUS   n[        U[        R                  5      (       a  UR	                  5       n[        U[        R                  5      (       a  UR	                  5       nUS   S   S:X  a  US S 2SS 24   nUS S 2SS 24   nOUnU R                  XvSS9nUc  U$ U R                  US S 2SS 24   5      nX4$ [        U[        R                  5      (       a  UR	                  5       nUR                  SS9nUR                  SS9nU R                  XvSS9nUc  U$ U R                  US S 2SS 24   5      nX4$ )Nr   r   rB   Fr   r   )rF   r   r   r   r   r   r   r   )	r   r   r   r   r|   preds_idr   r   rG   s	            r(   r   NRTRLabelDecode.__call__  sG   u:?QxHqJ(FMM22#>>+*fmm44'--/
{1~"$QUO	'12.
$	;;y%;PD}KKae-E { %//!,I*J;;y%;PD}KKae-E{r8   c                     / SQU-   nU$ )N)r   <unk><s></s>r;   r<   s     r(   r    NRTRLabelDecode.add_special_char  s    :^Kr8   c                    / n[        U5      n[        U5       H  n/ n/ n[        [        X   5      5       Hf  n	 U R                  [        X   U	   5         n
U
S:X  a    OAUR	                  U
5        Ub  UR	                  X&   U	   5        MU  UR	                  S5        Mh     SR                  U5      nUR	                  U[        R                  " U5      R                  5       45        M     U$ !    M  = f)rV   r  rB   r,   )	rF   rZ   r   r   r   r1   rD   r\   r]   )r   r^   r_   r`   rb   rd   re   rh   ri   r   char_idxrG   s               r(   r   NRTRLabelDecode.decode  s    _
z*IIIS!678#~~c*2G2L.MNH v%  *($$Y%9#%>?$$Q' 9 779%Dbggi&8&?&?&ABC! +" s   C,,C1r;   )NTr:   ro   )rp   rq   rr   rs   rt   r)   r   r   r   ru   r   r   s   @r(   r  r    s     3S8 r8   r  c                   @   ^  \ rS rSrSrSU 4S jjrSS jrS rSrU =r	$ )	ViTSTRLabelDecodei  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r  r)   r{   s       r(   r)   ViTSTRLabelDecode.__init__      /0CTr8   c                 *   [        U[        R                  5      (       a  US S 2SS 24   R                  5       nOUS S 2SS 24   nUR	                  SS9nUR                  SS9nU R                  XVSS9nUc  U$ U R                  US S 2SS 24   5      nXr4$ )NrB   r   r   Fr   r   r   s           r(   r   ViTSTRLabelDecode.__call__  s    eV]]++!QR%L&&(E!QR%LELLaL(	YYAY&
{{9e{L=KE!QR%L){r8   c                     SS/U-   nU$ )Nr  r  r;   r<   s     r(   r   "ViTSTRLabelDecode.add_special_char  s    >9r8   r;   ro   r:   r   r   s   @r(   r  r    s    3U r8   r  c                   @   ^  \ rS rSrSrSU 4S jjrSS jrS rSrU =r	$ )	ABINetLabelDecodei  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r  r)   r{   s       r(   r)   ABINetLabelDecode.__init__  r  r8   c                 L   [        U[        5      (       a  US   S   R                  5       nO2[        U[        R                  5      (       a  UR                  5       nOUnUR                  SS9nUR                  SS9nU R                  XVSS9nUc  U$ U R                  U5      nXr4$ )Nalignr-   r   r   Fr   )r   r   r   r   r   r   r   r   r   s           r(   r   ABINetLabelDecode.__call__  s    eT""'N2&,,.Ev}}--KKMEELLaL(	YYAY&
{{9e{L=KE"{r8   c                     S/U-   nU$ Nr  r;   r<   s     r(   r   "ABINetLabelDecode.add_special_char       N2r8   r;   ro   r:   r   r   s   @r(   r  r    s    3U  r8   r  c                   6   ^  \ rS rSrSrSU 4S jjrS rSrU =r$ )SPINLabelDecodei  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r  r)   r{   s       r(   r)   SPINLabelDecode.__init__  r   r8   c                 b    SU l         SU l        UnU R                   /U R                  /-   U-   nU$ r   r   r<   s     r(   r    SPINLabelDecode.add_special_char  s6    ',,4<<.8>Ir8   r   ro   )	rp   rq   rr   rs   rt   r)   r   ru   r   r   s   @r(   r  r    s    3S r8   r  c                   D   ^  \ rS rSrSrSU 4S jjrSS jrSS jrSrU =r	$ )	VLLabelDecodei  r   c                    > [         [        U ]  X5        UR                  SS5      U l        [        U R                  5      S-   U l        g )Nr  r  rB   )rz   r  r)   r  r  rF   r   nclassr{   s       r(   r)   VLLabelDecode.__init__  s<    mT+,?P%zz*;R@$..)A-r8   c                 @   / nU R                  5       n[        U5      n[        U5       H  n[        R                  " [        X   5      [
        S9nU(       a  X   SS X   SS :g  USS& U H  n	XU   U	:g  -  nM     X   U    V
s/ s H  n
U R                  U
S-
     PM     nn
Ub  X'   U   nOS/[        U5      -  n[        U5      S:X  a  S/nSR                  U5      nUR                  U[        R                  " U5      R                  5       45        M     U$ s  sn
f rU   )rY   rF   rZ   rD   r[   r.   r   r1   r   r\   r]   )r   r^   r_   r`   rb   rc   rd   re   rH   rf   rg   rh   ri   rG   s                 r(   r   VLLabelDecode.decode!  s9   002_
z*IJ$9 :$GI" * 5ab 9Z=RSVTV=W W	!"!/	2mCC	 "0
  *4Y??G w{+?   $%0;	C#i.0	9~"C	779%Dbggi&8&?&?&ABC' +( s   Dc           
      4   [        U5      S:X  Ga  Uu  pgUR                  S   nU R                  n	U R                  n
[        U[        R
                  5      (       d  [        R                  " USS9n[        R                  " XU R                  /UR                  S9n[        R                  " U/UR                  S9nSn[        U
5       Hw  nSU;   d  M  X:  d  M  XmS S 2S S 24   nXU'   UR                  S5      S   R                  SS9n[        U5       H!  nUU   S:X  d  M  UU   S:X  d  M  US-   UU'   M#     US-  nMy     [        SU5       H  n[        UU   5      S:X  d  M  XU'   M     Sn[        R                  " [        UR                  5       5      U R                  /UR                  S9n[        SU5       H(  n[        UU   5      nUSU2US S 24   UUUU-   & UU-  nM*     UnUnO@US   nUn[        R                  " [!        UU5       VVs/ s H  u  nnUS U PM     snn5      n/ n[        U[        R
                  5      (       d  [        R                  " USS9n["        R$                  " USS9n[        SUR                  S   5       GH]  nUS:X  a  Sn[        UU   5      nO>[        US U R                  5       5      n[        US U R                  5       UU   -   5      nUUU R                  S5      S   S S 2S4   R'                  5       nSR)                  U Vs/ s H6  nUS:  a+  U[        U R*                  5      ::  a  U R*                  US-
     OSPM8     sn5      nUUU R                  S5      S   S S 2S4   n[        R,                  " [        R.                  " U5      R                  5       UR                  S   S	-   -  5      nUR1                  U[3        U5      45        GM`     Uc  U$ U R5                  U5      nUU4$ s  snnf s  snf )
Nr   rB   float32rW   )r&  rX   r   r   r,   gư>)rF   r&  r  r   r   r   	to_tensorzerosr  rX   rZ   topksqueezer   r   concatzipFsoftmaxr]   r1   r   explogr   floatr   )r   r   r   r   r   r|   text_prexblenTextnstepsout_res
out_lengthnow_step_
tmp_resultjstartr   r&   
cur_lengthnet_outtlrG   rG  r   r   r   
preds_textr   s                                  r(   r   VLLabelDecode.__call__<  s   u:?KHq!A**G))Fh66!++HIFll'dkk)B!''RGQCqww?JH6]
?x'8!)Aq.!9J(2H%!+!3A!6!>!>A!>!FJ"1X%a=A-*Q-12D,4qLJqM & MH # 1a[z!}%*$*qM ! E\\:>>+,dkk:!''F 1a[ A/
5<Qz\1a=O5Puuz12# ! GF AhGFmms7F7K$L7Ktq!QrU7K$LMG'6==11&&wi@G))G!,q&,,q/*AAv	fQi.r
 01	fRajnn.:;	'277:1=adCJJLI  )  ) 7sc$...A'A sQw/   )	J !7388;A>q!tDJ

:&**,
0@0@0Cd0JKJ KKU:%678- +. =KE"U{? %Ms   P
'=P
)r  r  ro   )NN
rp   rq   rr   rs   rt   r)   r   r   ru   r   r   s   @r(   r  r    s    3.
6F Fr8   r  c                   D   ^  \ rS rSrSrSU 4S jjrSS jrSS jrSrU =r	$ )	CANLabelDecodei  -Convert between latex-symbol and symbol-indexc                 ,   > [         [        U ]  X5        g r:   )rz   r  r)   r{   s       r(   r)   CANLabelDecode.__init__  r   r8   c                 d   / n[        U5      n[        U5       H  nX   R                  S5      nX   S U R                  5       nU Vs/ s H  oR                  U   PM     n	n/ n
Ub  X%   S [        U	5       R                  5       n
UR                  SR                  U	5      U
/5        M     U$ s  snf )Nr   r   )rF   rZ   argminr]   r   r   r1   )r   r^   r   rb   rd   re   seq_endidx_listr   symbol_listr7  s              r(   r   CANLabelDecode.decode  s    _
z*I +2215G!,Xg6==?H:BC(3>>#.(KCE%"-.@K0@AHHJ 5u=> +  Ds   B-c                     Uu  n    nUR                  SS9nU R                  U5      nUc  U$ U R                  U5      nX4$ r  )r   r   )	r   r   r   r   r|   	pred_probr  r   rG   s	            r(   r   CANLabelDecode.__call__  sQ    "	1a$$!$,	{{9%=KE"{r8   r;   ro   r:   r  r   s   @r(   r  r    s    7R r8   r  c                   @   ^  \ rS rSrSrSU 4S jjrSS jrS rSrU =r	$ )	CPPDLabelDecodei  r   c                 ,   > [         [        U ]  X5        g r:   )rz   r  r)   r{   s       r(   r)   CPPDLabelDecode.__init__  r   r8   c                    [        U[        5      (       aE  [        US   [        5      (       a  US   S   S   R                  5       nOUS   R                  5       n[        U[        R
                  5      (       a  UR                  5       nOUnUR                  SS9nUR                  SS9nU R                  XVSS9nUc  U$ U R                  U5      nXr4$ )Nr-   r  r   r   Fr   )	r   r   r   r   r   r   r   r   r   r   s           r(   r   CPPDLabelDecode.__call__  s    eU##%)T**b	'*2.446b	)eV]]++KKMEELLaL(	YYAY&
{{9e{L=KE"{r8   c                     S/U-   nU$ r  r;   r<   s     r(   r    CPPDLabelDecode.add_special_char  r  r8   r;   ro   r:   r   r   s   @r(   r  r    s    3S$ r8   r  c                   B   ^  \ rS rSrSrU 4S jrS rS rSS jrSr	U =r
$ )	LaTeXOCRDecodei  r  c                 b   > SSK Jn  [        [        U ]  5         UR                  U5      U l        g )Nr   	Tokenizer)
tokenizersr  rz   r  r)   	from_file	tokenizer)r   rec_char_dict_pathr|   TokenizerFastr}   s       r(   r)   LaTeXOCRDecode.__init__  s&    9nd,.&001CDr8   c                   ^ SnSnSn[         R                  " X!5       Vs/ s H  oUS   R                  SS5      PM     snm[         R                  " UU4S jU5      nUn Un[         R                  " SU< S	U< S
3SU5      n[         R                  " SU< S	U< S
3SU5      n[         R                  " SU< S	U< S
3SU5      nXa:X  a   U$ Mo  s  snf )N1(\\(operatorname|mathrm|text|mathbf)\s?\*? {.*?})[a-zA-Z][\W_^\d]r   r   r,   c                 8   > [        TR                  S5      5      $ rl   strpopmatchnamess    r(   <lambda>-LaTeXOCRDecode.post_process.<locals>.<lambda>      3uyy|+<r8   (?!\\ )()\s+?()\1\2(r/   findallreplacerS  r   stext_regletternoletterr  newsr  s          @r(   post_processLaTeXOCRDecode.post_process      G02

80GH0G11c2&0GHFF8<a@A66HhGRSTD66HfEwPTUD66VX>NDy  I   Cc                    [        UR                  5      S:X  a	  US S S 24   nU Vs/ s H  o R                  R                  U5      PM     nnU Vs/ s Hq  nSR	                  UR                  S5      5      R                  SS5      R                  SS5      R                  SS5      R                  SS5      R                  5       PMs     nnU Vs/ s H  o`R                  U5      PM     sn$ s  snf s  snf s  snf )NrB   r,   r      Ġz[EOS]z[BOS]z[PAD])	rF   r&  r  r   r1   splitr  r   r  )r   tokenstokdecdetokdec_str_listdec_strs          r(   r   LaTeXOCRDecode.decode  s    v||!D!G_F5;<Vc~~$$S)V< 
  GGEKK$%WT3WWb!WWb!WWb!UW  	 
 ;GG,w!!'*,GG =
 Hs   $C,A8C1C6c                    US:X  a5  [         R                  " UR                  SS95      nU R                  U5      nO%U R                  [         R                  " U5      5      nUc  U$ U R                  [         R                  " U5      5      nXr4$ Ntrainr   r   )rD   r   r   r   r   r   r   moder   r|   r   rG   s           r(   r   LaTeXOCRDecode.__call__  sn    7?1!56I;;y)D;;rxx/D=KBHHUO,{r8   )r  Neval)rp   rq   rr   rs   rt   r)   r  r   r   ru   r   r   s   @r(   r  r    s!    7E H	 	r8   r  c                       \ rS rSr/ SQrS rSS\4S jjrS r\	S 5       r
\	S 5       r\	S	 5       rSS
\4S jjrS rS\4S jrS rS rSS jrSrg)UniMERNetDecodei  )	bos_token	eos_token	unk_token	sep_token	pad_token	cls_token
mask_tokenadditional_special_tokensc                 N   SSK Jn  SSK Jn  SU l        SU l        SU l        SU l        S U l        S U l        S U l	        / U l
        / SQU l        S	U l        S
U l        SU l        SU l        SU l        S
U l        SU l        SU l        S U l        [(        R*                  R-                  US5      n[(        R*                  R-                  US5      nUR/                  U5      U l        0 n0 nUGb  [3        USS9 n	[4        R6                  " U	5      n
SU
;   a|  U
S   R9                  5        He  u  p[;        U[<        5      (       a  U" S0 UD6n[;        X5      (       a  X[?        U5      '   X[A        U5      '   MN  [C        SURD                   S35      e   XzS'   U
RG                  S0 5      n[I        UR9                  5       S S9 VVs/ s H  u  pX;  d  M  UPM     nnnU RK                  U5      n[M        URO                  5       5      U Vs/ s H  n[A        U5      PM     sn-   nUU RP                   Vs/ s H  nUU;  d  M  X;  d  M  UPM     sn-  n[S        U5      S:  a  S n/ nU RT                  nU Hr  n[;        X5      (       a!  URV                  =(       d    [A        U5      U;   O[A        U5      U;   nUb  UU:X  a  URY                  U5        OU R[                  UUS9  U/nUnMt     U(       a  U R[                  UUS9  S S S 5        g g s  snnf s  snf s  snf ! , (       d  f       g = f)Nr   r  )
AddedTokenr  r  r  <pad>)	input_idstoken_type_idsattention_maski   rB   r   rightztokenizer.jsonztokenizer_config.jsonr   )encodingadded_tokens_decoderzFound a zV in the saved `added_tokens_decoder`, should be a dictionary or an AddedToken instancec                     U S   $ rl   r;   )r  s    r(   r  *UniMERNetDecode.__init__.<locals>.<lambda>8  s    AaDr8   r   )special_tokensr;   ).r  r  r/  
_unk_token
_bos_token
_eos_token
_pad_token
_sep_token
_cls_token_mask_token_additional_special_tokensmodel_input_namesmax_seq_lenpad_token_idbos_token_ideos_token_idpadding_sider*  pad_token_type_idpad_to_multiple_ofospathr1   r  r  r   jsonloaditemsr   r   r   r  r5  r}   r  sortedadded_tokens_encoderr   keysall_special_tokens_extendedrF   all_special_tokensspecialr   _add_tokens)r   r  r|   r  r/  fast_tokenizer_filetokenizer_config_filer6  added_tokens_maptokenizer_config_handleinit_kwargsr   tokenr,  tokens_to_addrQ  encoderis_last_specialr  r:  
is_specials                        r(   r)   UniMERNetDecode.__init__  sX   
 	:)! !*,'!R# !""& ggll+=?OP " 7!
 '001DE! ,%("ii(?@)[8&12H&I&O&O&Q
%eT22$.$7$7E%e88=BS:;@SZ8","*5??*;  <R  !S#  'R 7K23'27Mr'R$ )/,224.)!) 8	 )  ! (,'@'@AU'V$388:;,9?,95CJM?  !%!A!A"!AG+ 050J !A" 
 }%)&*OF%)%<%<N!.  *%<< #]]Jc%jN.J!$U~!= #
 +2o6S"MM%0 ,,VO,T&+WF*4 "/ (((Pe  -$!?"9 sJ   5CL LL0LLL.
L<LL	B3LL
L$returnc                 |    U(       a  U R                   R                  U5      $ U R                   R                  U5      $ r:   )r  add_special_tokens
add_tokens)r   
new_tokensr:  s      r(   rV  UniMERNetDecode._add_tokensX  s/    >>44Z@@~~((44r8   c                     [        UR                  5       S S9 VVs0 s H  u  p#UR                  U_M     snn$ s  snnf )Nc                     U S   $ rl   r;   )items    r(   r  6UniMERNetDecode.added_tokens_encoder.<locals>.<lambda>a  s	    dSTgr8   r9  )rP  rO  content)r   r6  vks       r(   rQ  $UniMERNetDecode.added_tokens_encoder^  sE     399;AUV
V IIqLV
 	
 
s   :c                 Z    U R                    Vs/ s H  n[        U5      PM     nnU$ s  snf r:   )rS  r  )r   r	  all_tokss      r(   rT  "UniMERNetDecode.all_special_tokensd  s/     %)$D$DE$DqCF$DE Fs   (c                 p   / n[        5       nU R                  R                  5        H  n[        U[        [
        45      (       a$  U Vs/ s H  n[        U5      U;  d  M  UPM     nnO[        U5      U;  a  U/O/ nUR                  [        [        U5      5        UR                  U5        M     U$ s  snf r:   )
setspecial_tokens_map_extendedvaluesr   r   r   r  updatemapextend)r   
all_tokensseenvaluer\  r]  s         r(   rS  +UniMERNetDecode.all_special_tokens_extendedj  s    
u55<<>E%$//49 TE5SZt=SE T+.u:T+ArKKC/0m, ?  !Us   	B3 B3c                 h    0 nU R                    H  n[        U SU-   5      nU(       d  M  X1U'   M!     U$ )Nr  )SPECIAL_TOKENS_ATTRIBUTESgetattr)r   set_attrattr
attr_values       r(   ru  +UniMERNetDecode.special_tokens_map_extendedw  s:    22D sTz2Jz!+ 3 r8   skip_special_tokensc                    [        U[        5      (       a  U R                  R                  U5      $ / nU HP  n[        U5      nU(       a  X@R                  ;   a  M&  UR                  U R                  R                  U5      5        MR     U$ r:   )r   r   r  id_to_tokenall_special_idsr   )r   r6  r  r  r,  s        r(   convert_ids_to_tokens%UniMERNetDecode.convert_ids_to_tokens  so    c3>>--c22EJE"u0D0D'DMM$..44U;<	 
 r8   c           	      L   SU R                   l        SU R                   l        SU R                   l        U Vs/ s H  o R	                  U5      PM     nn[        [        U5      5       H  n[        [        [        X4   5      5      5       H  nX4   U   c  SX4   U'   X4   U   R                  SS5      R                  5       X4   U'   X4   U   U R                   R                  U R                   R                  U R                   R                  4;   d  M  X4   U	 M     M     U$ s  snf )Nr  r  r0  r,   r  r   )
r  r&  r'  r*  r  rZ   rF   reversedr  r   )r   r  r  toksr  r&   s         r(   
detokenizeUniMERNetDecode.detokenize  s    #( #) #* ;AB6C**3/6Bs4y!AeCL1271:%!#DGAJ!WQZ//c:@@B
71:000000 
 3 "  Cs   D!c                 :   / nU Hm  n[         R                  " US:H  5      n[        U5      S:  a  [        US   S   5      nUS US-    nUR	                  U R
                  R                  USS95        Mo     U Vs/ s H  oPR                  U5      PM     nnU$ s  snf )Nr   r   rB   T)r  )rD   argwhererF   r   r   r  r   r  )r   	token_idsgenerated_texttok_idr   rG   s         r(   	token2strUniMERNetDecode.token2str  s    Fkk&A+.G7|agajm,'A+.!!%%f$%G   ?MMnd++D1nM Ns   :Bc                   ^ SnSnSn[         R                  " X!5       Vs/ s H  oUS   R                  SS5      PM     snm[         R                  " UU4S jU5      nUn Un[         R                  " SU< S	U< S
3SU5      n[         R                  " SU< S	U< S
3SU5      n[         R                  " SU< S	U< S
3SU5      nXa:X  a   U$ Mo  s  snf )Nr  r  r  r   r   r,   c                 8   > [        TR                  S5      5      $ rl   r  r  s    r(   r  +UniMERNetDecode.normalize.<locals>.<lambda>  r  r8   r   r  r  r  r  r  r  s          @r(   	normalizeUniMERNetDecode.normalize  r  r  c                 D    SSK Jn  U" U5      nU R                  U5      nU$ )Nr   )fix_text)ftfyr  r  )r   rG   r  s      r(   r  UniMERNetDecode.post_process  s!    !~~~d#r8   Nc                    US:X  a5  [         R                  " UR                  SS95      nU R                  U5      nO%U R                  [         R                  " U5      5      nUc  U$ U R                  [         R                  " U5      5      nXr4$ r  )rD   r   r   r  r  s           r(   r   UniMERNetDecode.__call__  sn    7?1!56I>>),D>>"((5/2D=Krxx/{r8   )rB  r<  r@  r=  rA  r>  r?  r;  rF  rG  rD  rC  rJ  r*  rE  rI  rH  r  )Fr"  )rp   rq   rr   rs   r  r)   r   rV  rQ  propertyrT  rS  ru  r.   r  r  r   r  r  r  r   ru   r;   r8   r(   r%  r%    s    	!TQl5s 5
  
 
 
  	d 	(d  	r8   r%  )rK  r   rD   r   	paddle.nnr   r  r/   rM  objectr   rw   r   r   r   r   r  r!  r=  rd  rp  rw  r  r  r  r  r  r  r  r  r%  r;   r8   r(   <module>r     sF   
   % 	 i iX' D" "JK( K\T' TnK( K\J' JZYO* YOxH"' H"VH") H"V" "J6( 6r;( ;| 0 6o i& iX' >o :1V 1hWf Wr8   