
    ړie                     F   U d Z ddlmZ ddlZddlZddlZddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZ ddlmZ ddlmZmZmZ dZej4                  Zej6                  Zej8                  Zej:                  Zej<                  Zej>                  Zej@                  Z ejB                  Z!ejD                  Z"ejF                  Z#ejH                  Z$ejJ                  Z%ejL                  Z& G d	 d
e
      Z' ejP                  d      Z)i Z*eee+e+f   e'f   e,d<   dddZ-d Z.ddddZ/d Z0d Z1dZ2 e0de2z   dz         Z3 e0de2z   dz         Z4 e0d      Z5 e0d      Z6d Z7 G d de
      Z8 G d de8      Z9 G d  d!      Z:d" Z;d# Z<d$d%d&e=d'ee+e+f   d(ee+e+f   d)ee9   fd*Z>d+ Z?dd$d,d-d.ee=   d'ee+e+f   d/ee+   d(ee+e+f   d)ee9   f
d0Z@d1 ZAeBd2k(  rNej                  d3   ZD eEeD      5 ZFeFj                         ZHddd        e>eH ed4      5      D ]
  ZI eJeI        yy# 1 sw Y   (xY w)6a  
This tokenizer has been copied from the ``tokenize.py`` standard library
tokenizer. The reason was simple: The standard library tokenizer fails
if the indentation is not right. To make it possible to do error recovery the
    tokenizer needed to be rewritten.

Basically this is a stripped down version of the standard library module, so
you can read the documentation there. Additionally we included some speed and
memory optimizations here.
    )absolute_importN)BOM_UTF8)	
NamedTupleTupleIteratorIterableListDictPatternSetAny)PythonTokenTypes)split_linesPythonVersionInfoparse_version_stringu   􏿿c                   z    e Zd ZU eed<   ee   ed<   ee   ed<   eeef   ed<   eed<   eeef   ed<   ee   ed<   y)	TokenCollectionpseudo_tokensingle_quotedtriple_quotedendpats
whitespacefstring_pattern_mapalways_break_tokensN)__name__
__module____qualname__r   __annotations__r   strr
        </tmp/pip-target-20744g9h/lib/python/parso/python/tokenize.pyr   r   +   sI    s8s8#w,c3h'S!r!   r   zutf-8_token_collection_cacheFcapturec                 J    |rJ d}| s|dz  }|dj                  |      z   dz   S )N(z?:|))join)r%   choiceskwargsstarts       r"   groupr.   :   s4    :E388G$$s**r!   c                      t        |  dz   S )N?)r.   )r+   s    r"   mayber1   C   s    '?S  r!   include_fstringonly_fstringc                     d }g d}dh}| rddg}|r|}t               }n||z  }n|r
t               S |D ]3  }t        j                  |      D ]  }|j                   ||              5 |S )Nc              3      K   t        j                  | D cg c]  }||j                         f c} D ]  }dj                  |        y c c}w w)N )
_itertoolsproductupperr*   )prefixcss      r"   different_case_versionsz5_all_string_prefixes.<locals>.different_case_versionsI   sF     ##f%Eq!'')n%EF 	A''!*	%Es   AA"A)brubrr7   ffr)setr8   permutationsupdate)r3   r4   r>   valid_string_prefixesresultrC   r;   ts           r"   _all_string_prefixesrK   H   s     2TF$K$%!UF!Q&!	u ( 6((0 	6A MM1!45	66
 Mr!   c                 J    t        j                  | t         j                        S N)recompileUNICODE)exprs    r"   _compilerR   e   s    ::dBJJ''r!   c                     	 t         t        |          S # t        $ r" t        |       xt         t        |       <   }|cY S w xY wrM   )r#   tupleKeyError_create_token_collection)version_inforI   s     r"   _get_token_collectionrX   i   sG    &u\':;; $\2	3l 34vs    (??z"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c                    d}t        |      }d}dt        z   dz   }d}d}d}d}t        ||||      }	d	}
t        d
d      t        |
      z   }d|
z   }t        ||      }t        d|dz         }t        |||	      }t	               }t        | }t        t	        d       }t	        dd      }t        | }d}d}d}d}t        |dz   |dz         }t        ddddddd      }d}g d }| d!k\  r|j                  d"d#       n|j                  d"d$       t        | }t        |||      }t        |d%z   t        d&d'      z   |d(z   t        d)d'      z         }||g} d*}!|r| j                  |t        |! z          t        d+g|  }"t        |d,      t        |"||||d,      z   }#i }$|D ]F  }%t        |      |$|%d&z   <   t        |      |$|%d)z   <   t        |      |$|%dz   <   t        |      |$|%dz   <   H t               }&t               }'i }(|D ]8  })d-D ]  }*|&j                  |)|*z           d.D ]  }*|'j                  |)|*z           : |D ]  })|!D ]
  }*|*|(|)|*z   <     d/}+t        |#      },t        |,|&|'|$||(t        |+            S )0Nz[ \f\t]*z	#[^\r\n]*u   ([A-Za-z_0-9-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r3   r2   z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])      r   z:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")ra   r`   r[   rZ   z\\(?:\r\n?|\n)|\Zr$   )ra   r`   )r[   rZ   );importclassdeftryexceptfinallywhilewithreturncontinuebreakdelpassglobalassertnonlocal)
rR   MAX_UNICODEr.   r1   rK   insertappendrE   addr   )-rW   
Whitespacer   CommentName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberpossible_prefixesStringPrefixStringPrefixWithFfstring_prefixesFStringStartSingleDoubleSingle3Double3TripleOperatorBracketspecial_argsSpecialFunnyContStrpseudo_extra_pool
all_quotesPseudoExtrasPseudoTokenr   _prefixr   r   r   rJ   quoteALWAYS_BREAK_TOKENSpseudo_token_compileds-                                                r"   rV   rV   ~   sL    J*%JG ;.6D*I#I$I2IiIyAI,HC-/16xAJ#h.H
H-K/w1FGJ:{I6F -.+,L3DIJ+DtT*+L !F F+G+G$u,.?%.GHF
 Ye+H
 G:LvAu%As#\"G(GW-E L#EEC!234 #EEC!2345G !&)'J  z0B!BC-B0ABL
D1lFE7D$GHK G$ 5!)&!1#!)&!1##+G#4% #+G#4% 	5 EMEM ) 	)Ea%i(	) " 	)Ea%i(	)	)  3 	3E-2E	*	33S %[1}mW'-@)A r!   c                   `    e Zd ZU eed<   eed<   eeef   ed<   eed<   edeeef   fd       Z	y)Tokentypestring	start_posr;   rk   c                     t        | j                        }t        |      dkD  r | j                  d   t        |      z   dz
  dfS | j                  d   | j                  d   t        | j                        z   fS N   r   )r   r   lenr   )selfliness     r"   end_poszToken.end_pos   sh    DKK(u:>>>!$s5z1A5q88>>!$dnnQ&7#dkk:J&JJJr!   N)
r   r   r   r   r   r   r   intpropertyr   r    r!   r"   r   r      sD    
KS#XKKsCx K Kr!   r   c                       e Zd Zd Zy)PythonTokenc                 T    d| j                  | j                  j                        z  S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r   )_replacer   namer   s    r"   __repr__zPythonToken.__repr__   s%    H499>>23 	4r!   N)r   r   r   r   r    r!   r"   r   r      s    4r!   r   c                   0    e Zd Zd Zd Zd Zd Zd Zd Zy)FStringNodec                 J    || _         d| _        d| _        d | _        d| _        y )Nr   r7   )r   parentheses_countprevious_lineslast_string_start_posformat_spec_count)r   r   s     r"   __init__zFStringNode.__init__   s+    
!" *." "#r!   c                 .    | xj                   dz  c_         y )Nr   )r   r   	characters     r"   open_parentheseszFStringNode.open_parentheses	  s    !#r!   c                 \    | xj                   dz  c_         | j                   dk(  rd| _        y y r   r   r   r   s     r"   close_parentheseszFStringNode.close_parentheses  s.    !#!!Q&%&D" 'r!   c                 2    t        | j                        dk(  S )Nr]   )r   r   r   s    r"   allow_multilinezFStringNode.allow_multiline  s    4::!##r!   c                 4    | j                   | j                  kD  S rM   r   r   s    r"   
is_in_exprzFStringNode.is_in_expr  s    %%(>(>>>r!   c                 @    | j                          xr | j                  S rM   )r   r   r   s    r"   is_in_format_speczFStringNode.is_in_format_spec  s    ??$$?)?)??r!   N)	r   r   r   r   r   r   r   r   r   r    r!   r"   r   r      s!    #$'$?@r!   r   c           
      j   t        |       D ]  \  }}|j                         }t        |      t        |      z
  }|j                  |j                        sIt        t        |j                  |||z   f||d | z         }	d}|j                  rJ | |d = |	dt        |j                        |z   fc S  d |dfS )Nr;   r7   r   )	enumeratelstripr   
startswithr   r   FSTRING_ENDr   )
fstring_stackr   line_nrcolumnadditional_prefixfstring_stack_indexnodelstripped_string
len_lstriptokens
             r"   _close_fstring_if_necessaryr     s    %.}%= ;!T!==?[3'7#88
&&tzz2

&:-.()<<	E !#****123"c$**o
:::; "A%%r!   c                 z   |d   }|j                         }|j                         r|rt        }nt        }n|rt        }nt
        }|j                  ||      }||j                  |fS |j                  s	||f|_        |j                  d      }	|D ]K  }
| |
j                     j                  |	      }|$|j                  d      d t        |
j                          }	M |}|t        |	      z  }|	j                  d      s|	j                  d      r|xj                  |	z  c_        d}	|	|fS |j                  |	z   }	|	|fS )Nr   
r7   )r   r   fstring_format_spec_multi_linefstring_format_spec_single_linefstring_string_multi_linefstring_string_single_linematchr   r   r.   r   r   endswith)r   r   linelnumpostosr   regexr   r   fstring_stack_node	end_matchnew_poss                r"   _find_fstring_stringr   .  sM   

C))+O
2E3E-E.EKKc"E}!!3&&%)3K![[^F+ I.445;;FC	 __Q'(G#.@.F.F*G)GHFI
 Gs6{G t 5f$ 7? ##f,7?r!   )r   r   )r   coderW   r   rk   c                8    t        | d      }t        |||      S )z0Generate tokens from a the source code (string).T)keepends)rW   r   )r   tokenize_lines)r   rW   r   r   s       r"   tokenizer   V  s     t,E%liPPr!   c                       fd}|S )zL
    A small helper function to help debug the tokenize_lines function.
    c               ?   F   K    | i |D ]  }t        |       |  y wrM   )print)argsr,   r   funcs      r"   wrapperz_print_tokens.<locals>.wrapperb  s,     4*6* 	E%LK	s   !r    )r   r   s   ` r"   _print_tokensr   ^  s    
 Nr!   T)indentsr   is_first_tokenr   r   c          
   #     ./K   ./fd}t        |      \  }}}}	}
}}d}dgd}d}d}d}d}d}|d   dz
  .g }| D ]m  }.dz  .d}t        |      }|rE|j                  t              rt        }|dd }t        |      }d|d   z  |z   }|d   }||d   z  }d	}|rNj	                  |      }|r0|j                  d      }t        t        ||d| z   |       d}d}n||z   }|z   }||k  s|r{|d
   }|j                         sBt        |	||.|      \  }}|r't        t        ||j                  d       d|_        X||k(  r||d }t        ||.||      \  }}}||z  }|| |r|} |D ]s  }!|!j                  }"|	|"   j	                  ||      }#|#'|#j                  d      }$t        |$      t        |"      z
  |z   t        |       k  s_|d| |$dt        |"        z   } u |j	                  | |      }%n|j	                  ||      }%|%rR||%j                  d      z   }d}|%j!                  d      \  }&}.|&f/|%j                  d      }'|'dk(  r|sJ |}|'d   }(n9|
j	                  ||      })||)j                            }(|)j                         }&.|&f/|rS|(dvrO|(dk7  s|%Hd	}|dk(  rA|s?|&}*|*d
   kD  r%t        t"        d/d       j%                  |*        ||*      E d{    |%s|
j	                  ||      })|r%|dk(  r |s ||)j                               E d{    |)j                         }d	}t        t&        ||   .|f||)j                  d      z          d}|dz  }|(|v s|(dk(  r 'dk7  r|'dk7  rt        t(        '/|       n
|%j                  d      '|v rD|s|r@g |dd d}t+        j                  d|d|&       }+|+ ||+j                               E d{    |'j-                         rt        t.        |'/|       nt1        |'/|      E d{    nt|(dv rIt3        d |D              r|j5                          |s|dk(  r|st        t6        '/|       n|'z   }d}n'|(dk(  r['j9                  d      s|'j9                  d      rJ |r.|d
   j                         rt        t&        |(/|       |&dz   }n||'z   }n'|v rP|	|'   }|j	                  ||      }|r,|j                  d      }||&| }'t        t        |'/|       n~/}||&d }|}|(|v s|'dd |v s|'dd |v rf|'d
   dv rJ.|&f}|	j;                  |(      xs* |	j;                  |'d         xs |	j;                  |'d         }||&d }|}Wt        t        |'/|       n|'|v r2|j%                  t=        ||'                t        t>        |'/|       n|(dk(  r||&d dv r||||&d z   z  }|'dv r|r|d
   jA                  |'       n}|dz  }nw|'dv r|r|d
   jC                  |'       n\|rZ|dz  }nT|'j                  d      rC|rA|d
   jD                  |d
   jF                  z
  dk(  r|d
   xjF                  dz  c_#        d}'|&dz   }t        tH        |'/|       ||k  rp |r8t        t&        ||       |j9                  d      s|j9                  d      rd}|r:|d
   }|j                  r)t        t        |j                  |j                  d       .|f},dd D ]&  }-jK                          t        tL        d|,d       ( t        tN        d|,|       y7 7 H7 7 Kw)a)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    c              3      K   | d   k  rR| d   kD  rt        t        d| fd       | d<   y j                          t        t        dd       | d   k  rQy y w)Nr   r7   )r   ERROR_DEDENTpopDEDENT)r-   r   r   sposs    r"   dedent_if_necessaryz+tokenize_lines.<locals>.dedent_if_necessaryz  sh     gbk!wr{"!,T5M2FF#KKMfb$33 gbk!s   AAAr   N
0123456789r7   Tr   ^Fr   r      z
#\.z...r]   z	[ \f\t]*$z
c              3   >   K   | ]  }|j                            y wrM   )r   ).0rC   s     r"   	<genexpr>z!tokenize_lines.<locals>.<genexpr>  s     F11,,..Fs   #r   r   )z\
z\
z\z([{z)]}r_   )(rX   r   r   BOM_UTF8_STRINGr   endr   STRINGr   r   FSTRING_STRINGr   r   r   r   r.   spanINDENTru   
ERRORTOKENNUMBERrN   isidentifierNAME_split_illegal_unicode_nameanyclearNEWLINEr   getr   FSTRING_STARTr   r   r   r   OPr   r   	ENDMARKER)0r   rW   r   r   r   r   r   r   r   r   r   r   r   paren_levelmax_numcharscontstrnew_liner;   r   r   r   r   endprogendmatchcontstr_startcontliner   r   restfstring_end_tokenquote_lengthstring_liner   r   r   end_match_stringpseudomatchr-   r   initialr   indent_startmr   indentr   r   s0     `                                           @@r"   r   r   k  s    4 	l+2L-0K#DHG HFQ<!D')M U;	4y/$3!ABx4y 1%,DA,CIaL D"N}}T*Hll1o!Gd4Cj0!6+ + !D.#d?Dj#B'~~'"6wtUY[^"_KFC)*F55 $&  .0* d{CDzE`!%FB!#4l |#$0++ "*7 V&.44E ' 4 4T3 ?I ,+4??1+=(/03u:=Cc+FVV*.t*7G#e*7U*UKV +00cB*00s;*[->->q-AA$&!(--a0
se}#))!,B;!M6(.% ("((s3uyy{+		e}G724;K^ !#M#(L#gbk1)&"dB??|42<@@@"((s3q 02599;???iik !S	D#;%A6  %'!q8#35C<EUN!&%v>>""1%1//]k')M!$"#KtFU|<A}#6quuw#???%%'%dE4@@:5$OOOF"FFF!'')K1$4]%gudFCC(.%C >>$/t8LLL ]2%6%A%A%C%j'4HH!)C(.%-'!%."==s3",,q/C sOE%feT6BB$(M"56lG#H M)"1I."1I.9& %)%KM&{{73  8w{{587L  8")++eAh"7 "56lG#H%feT6BB--$$[1DU1K%LM!-fEEDT%&\5O%O!Vd56l%::!E>$%b)::5A#q(e^$%b);;EB&'1,K%%c*})"-??'+==>ABC ""%771<7E!)C!"eT6::m Dj?U;n *g}fEED!W%5%5d%;HB 2 2))   DjG !"+ 3&"gr223 iW.?
@@O A
 @* @ Pse   C,]92B)]97]9D]9-]-.:]9(]0)C]90]31:]9+]6,I?]9-C]90]93]96]9c              #   D  	K   	fd}dd|	t        |       D ]p  \  }}r3|j                         r |        |dd|d   |d   |z   f	5|z  ;|z   }|j                         r|Sr |        d|d   |d   |z   f	|dr r
 |        y y w)Nc                  8    t        rt        nt               S rM   )r   r  r  )found
is_illegalr   r;   s   r"   create_tokenz1_split_illegal_unicode_name.<locals>.create_token  s    :uc6RRr!   r7   Fr   r   T)r   r  )
r   r   r;   r2  ichar	new_foundr0  r1  r   s
     `    @@@r"   r  r    s     S EJ
CU# "4  ""n$"
lIaL1$44I%%'!&.(F#A,	!q(88C!
)", n s   BB __main__r   z3.10)rW   )K__doc__
__future__r   sysrN   	itertoolsr8   codecsr   typingr   r   r   r   r	   r
   r   r   r   parso.python.tokenr   parso.utilsr   r   r   rs   r  r  r  r  r  r  r   r  r  r   r  r  r   r   decoder	  r#   r   r   r.   r1   rK   rR   rX   unicode_character_namer   r   r   r   rV   r   r   r   r   r   r   r   r   r   r  r   argvpathopenrC   readr   r   r   r    r!   r"   <module>rE     s  	 ' 
 	     0 L L 		 	 		 	 

"
"		 	 		 	 &&	((
,, ..!00**"j " "(//'*BD eCHo>? D " +!
 -2 :( ? %2234  %225KK  #++K"L !))!4 iXKJ K4% 4@ @:&$%R OUQ
Q %c3hQ<A#s(OQkQ	" !'ZAC=ZA S/ZA #Y	ZA
 S#XZA kZAzB z88A;D	d qvvx $-A&-IJ e  s   HH 