
    x-ju                     n    d dl Z d dlmZmZ  G d d          Z G d de          Z G d d          ZdS )	    N)Enumautoc                       e Zd Zd Zd ZdS )Tokenc                 "    || _         || _        d S Ntypevalue)selfr
   r   s      l/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddle/distributed/flex_checkpoint/aoa/lexer.py__init__zToken.__init__   s    	


    c                 (    d| j          d| j        dS )NzToken(z, )r	   )r   s    r   __repr__zToken.__repr__   s    4	44TZ4444r   N)__name__
__module____qualname__r   r    r   r   r   r      s2          5 5 5 5 5r   r   c                       e Zd Z e            Z e            Z e            Z e            Z e            Z e            Z	 e            Z
 e            Z e            Z e            Z e            ZdS )	TokenTypeN)r   r   r   r   
IDENTIFIERNUMBERCOLONLBRACKETRBRACKETCOMMARARROWSTRINGEQUALNEWLINEEOFr   r   r   r   r      s        JTVVFDFFEtvvHtvvHDFFETVVFTVVFDFFEdffG
$&&CCCr   r   c                   6    e Zd Zg dZddZd Zd Zd Zd ZdS )	Lexer))r   z->)r!   =)r   :)r   z\[)r   z\])r   ,)r   z\d+)r    z"[^"]*"|\'[^\']*\')r   z[A-Za-z_][A-Za-z\.\$\_\*\d\^T]*)SKIPz[ \t]+)r"   z[\r\n]+)MISMATCH.Nc                     ddl m} d |j         D             | _         t          j        d                    d | j        D                                 j        | _        || _        || _	        d S )N   )macro_registryc                 \    g | ])}t          |                                          d          *S )r-   )listvalues).0ds     r   
<listcomp>z"Lexer.__init__.<locals>.<listcomp>=   s-    JJJqtAHHJJ''*JJJr   |c              3   .   K   | ]\  }}d | d| dV  dS )z(?P<>r   Nr   )r2   nameregexs      r   	<genexpr>z!Lexer.__init__.<locals>.<genexpr>?   sM        D% 't&&e&&&     r   )
macrosr.   recompilejointoken_specificationmatch	get_tokencontext	traceback)r   rB   rC   r.   s       r   r   zLexer.__init__:   s    ******JJN4IJJJHH  #'#;    
 

  	 "r   c                    d}|                      ||          }g }|                    d          s|dz  }||j        }|                                }|dk    rnI|dk    rt	          d|d|           |                    t          t          |         |                     |                                }|                      ||          }||S )Nr   
r)   r*   zUnexpected character z at position )	rA   endswith	lastgroupgroupRuntimeErrorappendr   r   end)r   textposmotokenskindr   s          r   tokenizezLexer.tokenizeG   s    ^^D#&&}}T"" 	DLDn<DHHJJEv~~##"GEGG#GG   eIdOU;;<<<&&((Cc**B n r   c                    t          |t                    r|g}g }|D ]g} ||                     |          || j                  }t          |t                    r|                    |           R|                    |           h|S r   )
isinstancestrrQ   rB   rJ   extend)r   
expressionmacronew_expressionexprresultss         r   apply_macrozLexer.apply_macro\   s    j#&& 	&$J 	/ 	/DeDMM$//t|DDG'3'' /%%g....%%g....r   c           
         g }t          |dd          }|D ]/}	  ||                     |          || j                  }n# t          t          t
          t          f$ r~}| j        rq| j                            |          }| j        	                    t          |          | |t          |          j                   | j                                          d }~ww xY wt          |t                    r|g}	nt          |          }	| j        r#|	|gk    r| j                            ||	|           |                    |	           1|S )Nr   rW   )error_messagestagechain
error_type)getattrrQ   rB   AssertionError
ValueErrorKeyErrorrI   rC   build_chain	add_errorrT   r
   r   printrS   r0   record_childrenrU   )
r   expressionsrW   new_expressions
macro_namerY   rZ   er_   results_lists
             r   apply_single_macro_to_allzLexer.apply_single_macro_to_allh   sj   UJ88
 	1 	1D%d 3 3T4<HH"J,G 
 
 
> + N66t<<EN,,&)!ff!+o##'77#3	 -    N((***
 '3'' - 'y#G}}~ D6))N22lJ   ""<0000s   %A  CA9CCc                 ,   | j         r'| j                             t          |                     |}| j        D ]}|                     ||          }t          |          | _        g }|D ]*}|                    |                     |                     +|S r   )rC   register_rootsr0   r;   rn   final_expressionsrU   rQ   )r   ri   current_expressionsrW   rO   rY   s         r   
all_tokenszLexer.all_tokens   s    > 	=N))${*;*;<<<)[ 	 	E"&"@"@#U# # "&&9!:!:' 	/ 	/DMM$----....r   r   )	r   r   r   r?   r   rQ   r[   rn   rs   r   r   r   r%   r%   *   sw          # # # #  *
 
 
  @    r   r%   )r<   enumr   r   r   r   r%   r   r   r   <module>ru      s    
			        5 5 5 5 5 5 5 5       l l l l l l l l l lr   