o
    6d8                     @   s~   d dl mZmZmZmZ d dlZd dlZd dlmZ dZ	e	r"d dl
Z
G dd deZe ZG dd deZdd	 Zd
d ZdS )    )absolute_importdivisionprint_functionunicode_literalsNFc                   @   s   e Zd ZdZdd ZdS )
MultiTokenzdObject used to monkeypatch ply.lex so that we can return multiple
    tokens from one lex operation.c                 C   s   |d j | _ || _d S )Nr   )typetokens)selfr    r
   kC:\Users\jesus\OneDrive\Desktop\erpjis_fastapi\backend\jisbackend\Lib\site-packages\stone/frontend/lexer.py__init__   s   
zMultiToken.__init__N)__name__
__module____qualname____doc__r   r
   r
   r
   r   r      s    r   c                   @   sZ  e Zd ZdZdZdd Zdd Zdd Zd	d
 ZdZ	e	d7 Z	e	d7 Z	e	d7 Z	e	d7 Z	e	d7 Z	e	d7 Z	e	d7 Z	dZ
dZdZdZdZdZdZdZdZdZg dZddd d!d"d#d$d%d&d'd(d)d*Ze	ee 7 Z	d+d, Zd-d. Zd/d0 Zd1d2 Zd3d4 Zd5d6 Zd7d8 Zd9d: Zd;d< Z d=d> Z!d?d@ Z"dAdB Z#dCdD Z$dEdF Z%dGdH Z&dIdJ Z'dKZ(dLdM Z)dNS )OLexerz'
    Lexer. Tokenizes stone files.
    ))WSIGNOREZ	inclusivec                 C   s.   d | _ d | _d | _td| _d | _g | _d S )Nzstone.stone.lexer)lextokens_queue
cur_indentlogging	getLogger_logger
last_tokenerrors)r	   r
   r
   r   r   "   s   
zLexer.__init__c                 K   s6   t j dd| i|| _ g | _d| _| j |d  dS )z
        Required by ply.yacc for this to quack (duck typing) like a ply lexer.

        :param str file_data: Contents of the file to lex.
        moduler   
Nr
   )r   r   r   input)r	   Z	file_datakwargsr
   r
   r   r   ,   s   zLexer.inputc                 C   s   | j r| j d| _| jS | j }t|tr(| j |j | j d| _| jS |du ro| j	dkro| jrK| jj
dvrKtdd| jj| jj}| j | | j	}tdd| jj| jj}| j |g|  d| _	| j d| _| jS || _| jS )zf
        Returns the next LexToken. Returns None when all tokens have been
        exhausted.
        r   N)NEWLINEZLINEr   r   DEDENT	)r   popr   r   token
isinstancer   extendr   r   r   _create_tokenlinenolexposappend)r	   rnewline_tokenZdedent_countZdedent_tokenr
   r
   r   r#   9   s2   

zLexer.tokenc                 C   s,   |  | 	 |  }|sdS | jd| q)z;Logs all tokens for human inspection. Useful for debugging.TzToken %rN)r   r#   r   debug)r	   datar#   r
   r
   r   testX   s   
z
Lexer.test)IDKEYWORDPATHDOT)r    INDENTr   )COMMAZEQLPARRPAR)BOOLEANFLOATZINTEGERNULLSTRING)LBRACKETRBRACKET)LBRACERBRACECOLON)Q)ATz\.z\[z\]=,z\?z\{z\}z\:@)alias
annotationannotation_typeattrsby
deprecateddocZexampleerrorextendsimport	namespacepatchroutestructunionunion_closedZ
ANNOTATIONZANNOTATION_TYPEATTRSZ
DEPRECATEDZBYZEXTENDSZIMPORTPATCHZROUTEZSTRUCTZUNIONZUNION_CLOSED)rF   rG   rH   rJ   rI   rM   rN   rP   rQ   rR   rS   rT   c                 C   s   |j d |S )z\(r   )lexerZ
push_stater	   r#   r
   r
   r   t_LPAR      zLexer.t_LPARc                 C   s   |j   |S )z\))rW   Z	pop_staterX   r
   r
   r   t_RPAR   s   
zLexer.t_RPARc                 C   s   |j dk|_ |S )z\btrue\b|\bfalse\btrue)valuerX   r
   r
   r   t_ANY_BOOLEAN   rZ   zLexer.t_ANY_BOOLEANc                 C   s
   t |_|S )z\bnull\b)	NullTokenr]   rX   r
   r
   r   
t_ANY_NULL   s   zLexer.t_ANY_NULLc                 C   s:   |j | jv r|j dkr| jr|S | j|j d|_|S |S )z[a-zA-Z_][a-zA-Z0-9_-]*rG   r0   )r]   KEYWORDSr   RESERVEDgetr   rX   r
   r
   r   t_ANY_ID   s   	zLexer.t_ANY_IDc                 C   s   |S )z\/[/a-zA-Z0-9_-]*r
   rX   r
   r
   r   
t_ANY_PATH   s   zLexer.t_ANY_PATHc                 C      t |j|_|S )z-?\d+(\.\d*(e-?\d+)?|e-?\d+))floatr]   rX   r
   r
   r   t_ANY_FLOAT   rZ   zLexer.t_ANY_FLOATc                 C   rf   )z-?\d+)intr]   rX   r
   r
   r   t_ANY_INTEGER   rZ   zLexer.t_ANY_INTEGERc                    s   d}|j  j|jd7  _|jdd }d}tdt|D ]'}|| }|r:|dkr-d}n|dkr3d}||7 }d}q|d	krAd}q||7 }qd
t| j   fdd| D }d	||_|S )z\"([^\\"]|(\\.))*\"r   r       ntr!   \ c                    s   g | ]	}|  d dqS )rm   rk   )replace).0lineZindentation_strr
   r   
<listcomp>  s    z&Lexer.t_ANY_STRING.<locals>.<listcomp>)
rW   r'   r]   countrangelen_indent_level_to_spaces_countr   
splitlinesjoin)r	   ro   escapedsZnew_stricZlines_without_indentationr
   ru   r   t_ANY_STRING  s,   

zLexer.t_ANY_STRINGc                 C   s   |j  j|jd7  _|jd }|dkrb|j j| dk}| o'|j j| dk}|s,|rXtdd|j|jt|j d }|j |_ | |}|rI|S |rX|rV|j	
d| |S |S |d8 }|dksdS dS )[#][^\n]*\n+r   rk   r   rq   r   N)rW   r'   r]   rw   r(   lexdatar&   ry   !_create_tokens_for_next_line_dentr   insert)r	   r#   r   Zis_full_line_commentZis_partial_line_commentr+   dent_tokensr
   r
   r   t_INITIAL_comment%  s0   
zLexer.t_INITIAL_commentc                 C   sN   |j  j|jd7  _tdd|j|jt|j d }|j |_ | | dS )r   r   r   rk   N)rW   r'   r]   rw   r&   r(   ry   _check_for_indent)r	   r#   r+   r
   r
   r   t_WSIGNORE_commentC  s   zLexer.t_WSIGNORE_commentc                 C   s<   |j  j|jd7  _| |}|r|jd| |S |S )\n+r   r   )rW   r'   r]   rw   r   r   r   )r	   r+   r   r
   r
   r   t_INITIAL_NEWLINEL  s   
zLexer.t_INITIAL_NEWLINEc                 C   s&   |j  j|jd7  _| | dS )r   r   N)rW   r'   r]   rw   r   )r	   r+   r
   r
   r   t_WSIGNORE_NEWLINEV  s   zLexer.t_WSIGNORE_NEWLINEc                 C   sr   |  |}|du s|dkrdS |dkrdnd}t|d|jd |jt|j }|gt| }|  j|7  _t|S )z
        Starting from a newline token that isn't followed by another newline
        token, returns any indent or dedent tokens that immediately follow.
        If indentation doesn't change, returns None.
        Nr   r3   r    r!   rk   )	_get_next_line_indent_deltar&   r'   r(   ry   r]   absr   r   )r	   r+   indent_deltaZ	dent_typeZ
dent_tokenr   r
   r
   r   r   [  s   
z'Lexer._create_tokens_for_next_line_dentc                 C   s6   |  |}|du s|dkrdS | jd|jjf dS )zw
        Checks that the line following a newline is indented, otherwise a
        parsing error is generated.
        Nrk   z-Line continuation must increment indent by 1.)r   r   r)   rW   r'   )r	   r+   r   r
   r
   r   r   p  s   
zLexer._check_for_indentc                 C   s   |j dks	J d|jt|j }|t|jjkrdS |jj|d tjdd }|s.dS |	 }t|}|dkr<dS |d dkrDdS t|| }|d dkr\| j
d|jjf dS |t| j }|d S )	aX  
        Returns the change in indentation. The return units are in
        indentations rather than spaces/tabs.

        If the next line's indent isn't relevant (e.g. it's a comment),
        returns None. Since the return value might be 0, the caller should
        explicitly check the return type, rather than rely on truthiness.
        r   z3Can only search for a dent starting from a newline.Nrk   r   #   zIndent is not divisible by 4.)r   r(   ry   r]   rW   r   splitoslineseplstripr   r)   r'   rz   r   )r	   r+   Znext_line_posrt   Zlstripped_lineZlstripped_line_lengthindentr   r
   r
   r   r     s.   	
z!Lexer._get_next_line_indent_deltaz 	c                 C   sR   | j d|jd |jj | jdt|jd d |jjf |j	d d S )NzIllegal character %r at line %dr   zIllegal character %s.urk   )
r   r,   r]   rW   r'   r   r)   reprr   skiprX   r
   r
   r   t_ANY_error  s   zLexer.t_ANY_errorN)*r   r   r   r   Zstatesr   r   r#   r.   r   Zt_DOTZ
t_LBRACKETZ
t_RBRACKETZt_EQZt_COMMAZt_QZt_LBRACEZt_RBRACEZt_COLONZt_ATra   rb   tuplevaluesrY   r[   r^   r`   rd   re   rh   rj   r   r   r   r   r   r   r   r   Zt_ignorer   r
   r
   r
   r   r      st    

	$	
&r   c                 C   s$   t  }| |_||_||_||_|S )z
    Helper for creating ply.lex.LexToken objects. Unfortunately, LexToken
    does not have a constructor defined to make settings these values easy.
    )r   ZLexTokenr   r]   r'   r(   )
token_typer]   r'   r(   r#   r
   r
   r   r&     s   r&   c                 C   s   | d S )Nr   r
   )r   r
   r
   r   rz     s   rz   )
__future__r   r   r   r   r   r   Zply.lexr   Z_MYPYtypingobjectr   r_   r   r&   rz   r
   r
   r
   r   <module>   s    	   