a ze%@sNddlmZddlmZmZmZmZmZmZmZm Z m Z m Z m Z m ZmZmZmZddlmZmZddlZddlmZmZddlmZmZddlZddlZddZGd d d eZGd d d eZd dZ GdddeZ!GdddeZ"GdddeZ#GdddeZ$GdddeZ%GdddeZ&GdddeZ'e(dkrJe)dS))support)tokenize _tokenize untokenizeNUMBERNAMEOPSTRING ENDMARKERENCODINGtok_namedetect_encodingopen Untokenizergenerate_tokensNEWLINE)BytesIOStringION)TestCasemock)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALSc Csg}t|}|ddv}|D]b\}}}}} |tkr:q|rT|tkrT|d|krTq t|}|d|dd|dd|d|q |S)Nz r Z10 Z13)len splitlinesr rr append) Ztoken_generatorZ source_stringresultZ num_linesZmissing_trailing_nltypetokenstartendliner$F/opt/bitninja-python-dojo/embedded/lib/python3.9/test/test_tokenize.pystringify_tokens_from_sources  (r&c@seZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)S)* TokenizeTestcCs<t|d}tt|j|}||dg|dS)Nutf-8z* ENCODING 'utf-8' (0, 0) (0, 0))rencoder&rreadline assertEqualrstriprselfsexpectedfrr$r$r%check_tokenize*s zTokenizeTest.check_tokenizecCsDtdd}tt|j}||djt||djtdS)Nxr(r) rr)listrr*r+rrr )r.r1tokensr$r$r%test_implicit_newline4sz"TokenizeTest.test_implicit_newlinecCsd|dd|ddd}t|j}|tdt|D]}q ~0x15z NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) OP '~' (1, 10) (1, 11) NUMBER '0x15' (1, 11) (1, 15) z2134568 != 1231515z NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '1231515' (1, 11) (1, 18) z(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '200000000' (1, 14) (1, 23) z0xdeadbeef != -1z NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) z0xdeadc0de & 12345z NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '12345' (1, 13) (1, 18) z0xFF & 0x15 | 1234z NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) r2r.r$r$r%test_intas       zTokenizeTest.test_intcCs4|dd|dd|dd|dddS) Nzx = 0z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0' (1, 4) (1, 5) zx = 0xfffffffffffz NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xfffffffffff' (1, 4) (1, 17) zx = 123141242151251616110z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151251616110' (1, 4) (1, 25) zx = -15921590215012591z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '15921590215012591' (1, 5) (1, 22) r<r=r$r$r% test_longs   zTokenizeTest.test_longcCsX|dd|dd|dd|dd|d d |d d |d ddS)Nz x = 3.14159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) z x = 314159.z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) z x = .314159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) z x = 3e14159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) z x = 3E123z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) z x+y = 3e-1230z NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) z x = 3.14e159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) r<r=r$r$r% test_floats      zTokenizeTest.test_floatcCsJdd}tD]}d|vrq ||||q tD]}||||q0dS)NcSs<t|d}t|jD]\}}}}}|tkr|SqdS)Nr(z invalid token)rr)rr*r)r/r1Ztoktyper r!r"r#r$r$r% number_tokens  z;TokenizeTest.test_underscore_literals..number_token()rr+rZassertNotEqual)r.rAZlitr$r$r%test_underscore_literalssz%TokenizeTest.test_underscore_literalscCs<|dd|dd|dd|dd|d d |d d |d d|dd|dd|dd|dd|dd|dd|dd|dd|dd |d!d"|d#d$|d%d&|d'd(|d)d*|d+d,|d-d.|d/d0|d1d2|d3d4dS)5Nzx = ''; y = ""a5 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "''" (1, 4) (1, 6) OP ';' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '=' (1, 10) (1, 11) STRING '""' (1, 12) (1, 14) zx = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '\'"\'' (1, 4) (1, 7) OP ';' (1, 7) (1, 8) NAME 'y' (1, 9) (1, 10) OP '=' (1, 11) (1, 12) STRING '"\'"' (1, 13) (1, 16) zx = "doesn't "shrink", does it"z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"doesn\'t "' (1, 4) (1, 14) NAME 'shrink' (1, 14) (1, 20) STRING '", does it"' (1, 20) (1, 31) zx = 'abc' + 'ABC'z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "'abc'" (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING "'ABC'" (1, 12) (1, 17) zy = "ABC" + "ABC"z NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"ABC"' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING '"ABC"' (1, 12) (1, 17) z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "r'abc'" (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING "r'ABC'" (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING "R'ABC'" (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING "R'ABC'" (1, 31) (1, 37) z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'r"abc"' (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING 'r"ABC"' (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING 'R"ABC"' (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING 'R"ABC"' (1, 31) (1, 37) zu'abc' + U'abc'z STRING "u'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "U'abc'" (1, 9) (1, 15) zu"abc" + U"abc"z STRING 'u"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'U"abc"' (1, 9) (1, 15) zb'abc' + B'abc'z STRING "b'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "B'abc'" (1, 9) (1, 15) zb"abc" + B"abc"z STRING 'b"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'B"abc"' (1, 9) (1, 15) z%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "bR'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Br'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "BR'abc'" (1, 30) (1, 37) z%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'bR"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Br"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'BR"abc"' (1, 30) (1, 37) z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "rB'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Rb'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "RB'abc'" (1, 30) (1, 37) z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'rB"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Rb"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) z "a\ de\ fg"z3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3) zu"a\ de"z/ STRING 'u"a\\\nde"' (1, 0) (2, 3) zrb"a\ d"z/ STRING 'rb"a\\\nd"' (1, 0) (2, 2) z """a\ b"""z0 STRING '"""a\\\nb"""' (1, 0) (2, 4) z u"""a\ b"""z1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4) zrb"""a\ b\ c"""z7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4) zf"abc"z/ STRING 'f"abc"' (1, 0) (1, 6) z fR"a{b}c"z/ STRING 'fR"a{b}c"' (1, 0) (1, 9) z f"""abc"""z0 STRING 'f"""abc"""' (1, 0) (1, 10) z f"abc\ def"z1 STRING 'f"abc\\\ndef"' (1, 0) (2, 4) z Rf"abc\ def"z2 STRING 'Rf"abc\\\ndef"' (1, 0) (2, 4) r<r=r$r$r% test_stringsT           zTokenizeTest.test_stringcCs(|dd|dd|dddS)Nz!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) zdef d01v_(a=1, *k, **w): passa NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) z&def d23(a: str, b: int=3) -> int: passa NAME 'def' (1, 0) (1, 3) NAME 'd23' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ':' (1, 9) (1, 10) NAME 'str' (1, 11) (1, 14) OP ',' (1, 14) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'int' (1, 19) (1, 22) OP '=' (1, 22) (1, 23) NUMBER '3' (1, 23) (1, 24) OP ')' (1, 24) (1, 25) OP '->' (1, 26) (1, 28) NAME 'int' (1, 29) (1, 32) OP ':' (1, 32) (1, 33) NAME 'pass' (1, 34) (1, 38) r<r=r$r$r% test_functions  zTokenizeTest.test_functioncCs|dddS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa NAME 'if' (1, 0) (1, 2) NUMBER '1' (1, 3) (1, 4) OP '<' (1, 5) (1, 6) NUMBER '1' (1, 7) (1, 8) OP '>' (1, 9) (1, 10) NUMBER '1' (1, 11) (1, 12) OP '==' (1, 13) (1, 15) NUMBER '1' (1, 16) (1, 17) OP '>=' (1, 18) (1, 20) NUMBER '5' (1, 21) (1, 22) OP '<=' (1, 23) (1, 25) NUMBER '0x15' (1, 26) (1, 30) OP '<=' (1, 31) (1, 33) NUMBER '0x12' (1, 34) (1, 38) OP '!=' (1, 39) (1, 41) NUMBER '1' (1, 42) (1, 43) NAME 'and' (1, 44) (1, 47) NUMBER '5' (1, 48) (1, 49) NAME 'in' (1, 50) (1, 52) NUMBER '1' (1, 53) (1, 54) NAME 'not' (1, 55) (1, 58) NAME 'in' (1, 59) (1, 61) NUMBER '1' (1, 62) (1, 63) NAME 'is' (1, 64) (1, 66) NUMBER '1' (1, 67) (1, 68) NAME 'or' (1, 69) (1, 71) NUMBER '5' (1, 72) (1, 73) NAME 'is' (1, 74) (1, 76) NAME 'not' (1, 77) (1, 80) NUMBER '1' (1, 81) (1, 82) OP ':' (1, 82) (1, 83) NAME 'pass' (1, 84) (1, 88) r<r=r$r$r%test_comparisonszTokenizeTest.test_comparisoncCs|dddS)Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) r<r=r$r$r% test_shiftszTokenizeTest.test_shiftcCs|dddS)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '1' (1, 17) (1, 18) OP '+' (1, 19) (1, 20) NUMBER '0x124' (1, 21) (1, 26) OP '+' (1, 27) (1, 28) NAME 'z' (1, 29) (1, 30) OP '+' (1, 31) (1, 32) NAME 'a' (1, 33) (1, 34) OP '[' (1, 34) (1, 35) NUMBER '5' (1, 35) (1, 36) OP ']' (1, 36) (1, 37) r<r=r$r$r% test_additiveszTokenizeTest.test_additivecCs|dddS)Nzx = 1//1*1/5*12%0x12@42a NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) OP '@' (1, 20) (1, 21) NUMBER '42' (1, 21) (1, 23) r<r=r$r$r%test_multiplicativesz TokenizeTest.test_multiplicativecCs|dd|dddS)Nz~1 ^ 1 & 1 |1 ^ -1a OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) z-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) r<r=r$r$r% test_unary(s zTokenizeTest.test_unarycCs|dddS)Nz/import sys, time x = sys.modules['time'].time()a NAME 'import' (1, 0) (1, 6) NAME 'sys' (1, 7) (1, 10) OP ',' (1, 10) (1, 11) NAME 'time' (1, 12) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'x' (2, 0) (2, 1) OP '=' (2, 2) (2, 3) NAME 'sys' (2, 4) (2, 7) OP '.' (2, 7) (2, 8) NAME 'modules' (2, 8) (2, 15) OP '[' (2, 15) (2, 16) STRING "'time'" (2, 16) (2, 22) OP ']' (2, 22) (2, 23) OP '.' (2, 23) (2, 24) NAME 'time' (2, 24) (2, 28) OP '(' (2, 28) (2, 29) OP ')' (2, 29) (2, 30) r<r=r$r$r% test_selectorMszTokenizeTest.test_selectorcCs|dddS)Nz @staticmethod def foo(x,y): passa OP '@' (1, 0) (1, 1) NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) OP '(' (2, 7) (2, 8) NAME 'x' (2, 8) (2, 9) OP ',' (2, 9) (2, 10) NAME 'y' (2, 10) (2, 11) OP ')' (2, 11) (2, 12) OP ':' (2, 12) (2, 13) NAME 'pass' (2, 14) (2, 18) r<r=r$r$r% test_methodcszTokenizeTest.test_methodcCs|dddS)Nzdef f(): if x passa_ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) INDENT '\t' (2, 0) (2, 1) NAME 'if' (2, 1) (2, 3) NAME 'x' (2, 4) (2, 5) NEWLINE '\n' (2, 5) (2, 6) INDENT ' \t' (3, 0) (3, 9) NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) r<r=r$r$r% test_tabstszTokenizeTest.test_tabscCs|dddS)Nu!Örter = 'places' grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "'places'" (1, 8) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "'green'" (2, 7) (2, 14) r<r=r$r$r%test_non_ascii_identifierssz'TokenizeTest.test_non_ascii_identifierscCs|dddS)Nu#Örter = u'places' grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "u'places'" (1, 8) (1, 17) NEWLINE '\n' (1, 17) (1, 18) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "U'green'" (2, 7) (2, 15) r<r=r$r$r% test_unicodeszTokenizeTest.test_unicodecCs|dd|dd|dd|dd|d d |d d |d d|dd|dd|dd|dd|dd|dd|dd|dd|dd |d!d"|d#d$|d%d&|d'd(|d)d*dS)+Nz async = 1z NAME 'async' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) za = (async = 1)a8 NAME 'a' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '(' (1, 4) (1, 5) NAME 'async' (1, 5) (1, 10) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) zasync()z NAME 'async' (1, 0) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) zclass async(Bar):passa< NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP '(' (1, 11) (1, 12) NAME 'Bar' (1, 12) (1, 15) OP ')' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 17) (1, 21) zclass async:passz NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP ':' (1, 11) (1, 12) NAME 'pass' (1, 12) (1, 16) z await = 1z NAME 'await' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) z foo.asyncz NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) zasync for a in b: passa; NAME 'async' (1, 0) (1, 5) NAME 'for' (1, 6) (1, 9) NAME 'a' (1, 10) (1, 11) NAME 'in' (1, 12) (1, 14) NAME 'b' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 18) (1, 22) zasync with a as b: passa< NAME 'async' (1, 0) (1, 5) NAME 'with' (1, 6) (1, 10) NAME 'a' (1, 11) (1, 12) NAME 'as' (1, 13) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'pass' (1, 19) (1, 23) z async.fooz NAME 'async' (1, 0) (1, 5) OP '.' (1, 5) (1, 6) NAME 'foo' (1, 6) (1, 9) asyncz/ NAME 'async' (1, 0) (1, 5) zasync #comment awaitz NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) COMMENT '#comment' (2, 0) (2, 8) NL '\n' (2, 8) (2, 9) NAME 'await' (3, 0) (3, 5) zasync ... awaitz NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) OP '...' (2, 0) (2, 3) NEWLINE '\n' (2, 3) (2, 4) NAME 'await' (3, 0) (3, 5) z async awaitz NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) NAME 'await' (2, 0) (2, 5) z foo.async + 1z NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '1' (1, 12) (1, 13) zasync def foo(): passa; NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NAME 'pass' (1, 17) (1, 21) zNasync def foo(): def foo(await): await = 1 if 1: await async += 1 a NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'def' (2, 2) (2, 5) NAME 'foo' (2, 6) (2, 9) OP '(' (2, 9) (2, 10) NAME 'await' (2, 10) (2, 15) OP ')' (2, 15) (2, 16) OP ':' (2, 16) (2, 17) NEWLINE '\n' (2, 17) (2, 18) INDENT ' ' (3, 0) (3, 4) NAME 'await' (3, 4) (3, 9) OP '=' (3, 10) (3, 11) NUMBER '1' (3, 12) (3, 13) NEWLINE '\n' (3, 13) (3, 14) DEDENT '' (4, 2) (4, 2) NAME 'if' (4, 2) (4, 4) NUMBER '1' (4, 5) (4, 6) OP ':' (4, 6) (4, 7) NEWLINE '\n' (4, 7) (4, 8) INDENT ' ' (5, 0) (5, 4) NAME 'await' (5, 4) (5, 9) NEWLINE '\n' (5, 9) (5, 10) DEDENT '' (6, 0) (6, 0) DEDENT '' (6, 0) (6, 0) NAME 'async' (6, 0) (6, 5) OP '+=' (6, 6) (6, 8) NUMBER '1' (6, 9) (6, 10) NEWLINE '\n' (6, 10) (6, 11) z)async def foo(): async for i in 1: passa NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'async' (2, 2) (2, 7) NAME 'for' (2, 8) (2, 11) NAME 'i' (2, 12) (2, 13) NAME 'in' (2, 14) (2, 16) NUMBER '1' (2, 17) (2, 18) OP ':' (2, 18) (2, 19) NAME 'pass' (2, 20) (2, 24) DEDENT '' (3, 0) (3, 0) zasync def foo(async): awaitah NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) NAME 'async' (1, 14) (1, 19) OP ')' (1, 19) (1, 20) OP ':' (1, 20) (1, 21) NAME 'await' (1, 22) (1, 27) z@def f(): def baz(): pass async def bar(): pass await = 2a NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) zFasync def f(): def baz(): pass async def bar(): pass await = 2a NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'f' (1, 10) (1, 11) OP '(' (1, 11) (1, 12) OP ')' (1, 12) (1, 13) OP ':' (1, 13) (1, 14) NEWLINE '\n' (1, 14) (1, 15) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) r<r=r$r$r% test_asyncs:            , %zTokenizeTest.test_asyncN)__name__ __module__ __qualname__r2r7r;r>r?r@rCrDrErFrGrHrIrJrKrLrMrNrOrQr$r$r$r%r'#s* %9('<& %  r'c@seZdZddZdS)GenerateTokensTestcCs0t|}tt|j|}|||dSN)rr&rr*r+r,rr-r$r$r%r2sz!GenerateTokensTest.check_tokenizeN)rRrSrTr2r$r$r$r%rUsrUcCsg}tt|dj}|D]T\}}}}}|tkrbd|vrb|tdftdftt |ftdfgq| ||fqt | dS)Nr(.DecimalrB)) rrr)r*rextendrrr reprrrdecode)r/rgtoknumtokval_r$r$r%decistmts rac@seZdZddZdS)TestMisccCsPddlm}d}|t|d|tt|d|tt||ddS)Nr)rXz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)ZdecimalrXr+raZ assertRegexr[eval)r.rXr/r$r$r% test_decistmts  zTestMisc.test_decistmtN)rRrSrTrdr$r$r$r%rbsrbc@s<eZdZddZddZddZddZd d Zd d Zd S)TestTokenizerAdheresToPep0263cCs,tjtjt|}t|t|ddS)Nrb)ospathjoindirname__file__ TestRoundtripcheck_roundtripr)r.filenamerhr$r$r% _testFilesz'TestTokenizerAdheresToPep0263._testFilecCsd}||dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtror.r1r$r$r%'test_utf8_coding_cookie_and_no_utf8_bomszETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomcCsd}|t|j|dS)Nz8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt assertRaises SyntaxErrorrorqr$r$r%&test_latin1_coding_cookie_and_utf8_bomszDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomcCsd}||dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtrprqr$r$r%"test_no_coding_cookie_and_utf8_bomsz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomcCsd}||dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtrprqr$r$r%$test_utf8_coding_cookie_and_utf8_bomszBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomcCs$|t|jd|t|jddS)Nz bad_coding.pyzbad_coding2.pyrsr=r$r$r%test_bad_coding_cookiesz4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN) rRrSrTrorrrvrwrxryr$r$r$r%res  rec@seZdZddZddZdS) Test_TokenizecsPd}|ddfdd}tt|dddd}d g}|||d dS) N "ЉЊЈЁЂ"r(Fcss dSdSdSNTr$r$firstr#r$r%r* szNTest_Tokenize.test__tokenize_decodes_with_specified_encoding..readlineencodingr4r{)rr)rr{zbytes not decoded with encoding)r)r5rr+)r.literalr*r6expected_tokensr$r~r%.test__tokenize_decodes_with_specified_encoding s  z.readlinerr4rz*string not tokenized when encoding is None)r5rr+)r.r*r6rr$rr%1test__tokenize_does_not_decode_with_encoding_nones z?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rRrSrTrrr$r$r$r%rzsrzc@seZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)S)*TestDetectEncodingcsdfdd}|S)Nrcs$tkrt}d7|SNr)r StopIteration)r#indexlinesr$r%r*1s  z1TestDetectEncoding.get_readline..readliner$r.rr*r$rr% get_readline/szTestDetectEncoding.get_readlinecCs>d}t||\}}||d||t|dddS)N) # something print(something) do_something(else) r()r rr+r5r.rrconsumed_linesr$r$r%test_no_bom_no_encoding_cookie:s z1TestDetectEncoding.test_no_bom_no_encoding_cookiecCs6d}t||\}}||d||ddgdS)N)s# something rr utf-8-sigrrr rr+rr$r$r%test_bom_no_cookieDs  z%TestDetectEncoding.test_bom_no_cookiecCs4d}t||\}}||d||dgdS)N)# -*- coding: latin-1 -*- rr iso-8859-1rrrr$r$r%test_cookie_first_line_no_bomOs z0TestDetectEncoding.test_cookie_first_line_no_bomcCs4d}t||\}}||d||dgdS)N)s# coding=utf-8 rrrs# coding=utf-8 rrr$r$r%&test_matched_bom_and_cookie_first_lineYs z9TestDetectEncoding.test_matched_bom_and_cookie_first_linecCs d}||}|tt|dS)N)s## vim: set fileencoding=ascii : rrrrtrur rr$r$r%.BunkcSs||_||_d|_dS)Nr)nameZ_lines_index)r.rrhr$r$r%__init__szDTestDetectEncoding.test_filename_in_exception..Bunk.__init__cs.|jtkrt|j}|jd7_|Sr)rrr)r.r#rr$r%r*s  zDTestDetectEncoding.test_filename_in_exception..Bunk.readlineN)rRrSrTrr*r$rr$r%Bunk srz.*{})rtrurr r*r8format)r.rhrZinsr$rr%test_filename_in_exceptions  ( z-TestDetectEncoding.test_filename_in_exceptioncCsTtd}tjd|d|ttdWdn1s:0Y||jdS)Ns #coding:xxxztokenize._builtin_open)Z return_valueZfoobar)rrZpatchrtrurZ assertTrueclosed)r.mr$r$r%test_open_error#s,z"TestDetectEncoding.test_open_errorN)rRrSrTrrrrrrrrrrrrrrrrrrrrr$r$r$r%r-s(              rc@s<eZdZddZddZddZddZd d Zd d Zd S) TestTokenizecsddl}tdfdd}fdd}dfdd}|j}|j}||_||_z,t|}|t|gdW||_||_n||_||_0|dS) Nrcs ddgfS)Nfirstsecondr$)r*rr$r%mock_detect_encoding1sz8TestTokenize.test_tokenize..mock_detect_encodingcs"|g}|}|r||q|SrV)r)r*routZ next_line) encoding_usedr$r%mock__tokenize4s z2TestTokenize.test_tokenize..mock__tokenizecs d7dkrdStS)Nrr})strr)r$)counterr$r% mock_readline@sz1TestTokenize.test_tokenize..mock_readline)rr1234)robjectr rr+r5)r.Ztokenize_modulerrrZorig_detect_encodingZorig__tokenizeresultsr$)rrrr% test_tokenize-s*    zTestTokenize.test_tokenizecCsfg}tdD]}|dj|dq |dd|}ttt|dj}| |dj ddS)Nizdef i{i}(): return {i})iZOK r() rangerrrir5rrr)r*r+string)r.bufrZtoksr$r$r%test_oneline_defsUs   zTestTokenize.test_oneline_defscGsttt|dj}t|}|t|d||t|djtt t |D]&}|t||djt||qV|t|d|jtt j |t|d|jtt j dS)Nr(rrrr)r5rrr)r*rr+r exact_typer rr rr )r.ZopstrZoptypesr6Z num_optypesrr$r$r%assertExactTypeEqualas   z!TestTokenize.assertExactTypeEqualcCs|dtjtj|dtjtj|dtj|dtj|dtj|dtj |dtj |dtj |d tj |d tj |d tj|d tj|d tj|dtj|dtj|dtj|dtjtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj |dtj!|dtj"|dtj#|d tj$|d!tj%|d"tj&|d"tj&|d#tj'|d$tj(|d%tj)|d&tj*|d'tj+|d(tj,|d)tj-|d*tj.|d+tj/|d,tj0|d-t1tjt2tj t1tjt2tjt1tjt2 |d.tjtj2tjtj2tjtj2tj|d/tjtjtj1tjtj2tjdS)0Nz()z[]:,;+r*/|&<>=rW%z{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z {1, 2, 3}z ^(x & 0x1))3rr LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE CIRCUMFLEX LEFTSHIFT RIGHTSHIFT DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL SLASHEQUAL PERCENTEQUAL AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUAL COLONEQUALELLIPSISRARROWATATEQUALrrr=r$r$r%test_exact_typeos zTestTokenize.test_exact_typecCs|dtjdS)Nz @ )rr r2r=r$r$r%%test_pathological_trailing_whitespacesz2TestTokenize.test_pathological_trailing_whitespacecCsd}tjtjtjtjtjtjg}ttt | dj }| t |djt ttdD]&}| t ||djt ||qZ| t |djt tjdS)Nz b = 1 #testr(rrr)r rrrrNLCOMMENTr5rrr)r*r+r rr rr )r.sourcerr6rr$r$r%5test_comment_at_the_end_of_the_source_without_newlines $zBTestTokenize.test_comment_at_the_end_of_the_source_without_newlineN) rRrSrTrrrr4r5r:r$r$r$r%r+s ( Arc@s$eZdZddZddZddZdS)UntokenizeTestcCsnt}d|_d|_|t}|dWdn1s<0Y||jjdd|t|jddS)Nr)rrrz'start (1,3) precedes previous end (2,2))rr) rprev_rowprev_colrt ValueErroradd_whitespacer+ exceptionargs)r.ucmr$r$r%test_bad_input_orders (z#UntokenizeTest.test_bad_input_ordercCsdt}d|_d|_g|_|d||jdgd|_|d||jgdt|ddS)Nr)rr\ r)rF)rEz\ \ rza b c \ c )rr<r=r6r?r+rlrm)r.rBr$r$r%test_backslash_continuations  z*UntokenizeTest.test_backslash_continuationcCst}tdf}tdf|g}||tg||jdgt}||t|gdt}||t|d||jd|tt|ddS)NZHellor(zHello sHello ) rrr compatiterr+r6rr)r.rBr r6r$r$r%test_iter_compats zUntokenizeTest.test_iter_compatN)rRrSrTrDrGrJr$r$r$r%r;s  r;c@sLeZdZddZddZddZddZd d Zd d Zd dZ ddZ dS)rlc Cst|tr|d}n|}|t|jddj}tt |}dd|D}t |}t|jddj}ddt |D}| ||t |} t| jddj} ddt | D} | | |dS)Nr(T)keependscSsg|]}|ddqSNrr$.0r:r$r$r% r}z1TestRoundtrip.check_roundtrip..cSsg|]}|ddqSrLr$rMr$r$r%rOr}cSsg|]}|ddqSrLr$rMr$r$r%rOr}) isinstancerr)readcloserIr__next__r5rrr+) r.r1coder*Ztokens5Ztokens2Z bytes_from2Z readline2Z tokens2_from2Z bytes_from5Z readline5Z tokens2_from5r$r$r%rms    zTestRoundtrip.check_roundtripcCs|d|d|dtd}t|d}||Wdn1sR0Y|d|d|d|d dS) Nzif x == 1: print(x) z # This is a comment # This also zif x == 1 : print(x) tokenize_tests.txtrfzoif x == 1: # A comment by itself. print(x) # Comment here, too. # Another comment. after_if = True zPif (x # The comments need to go in the right place == 1): print('x==1') zclass Test: # A comment here # A comment with weird indent after_com = 5 def x(m): return m*5 # a one liner def y(m): # A whitespace after the colon return y*4 # 3-space indent zttry: import somemodule except ImportError: # comment print('Can not import' # comment2 )else: print('Loaded') )rmrfindfiler)r.fnr1r$r$r%test_roundtrip s     (   zTestRoundtrip.test_roundtripcCs|ddS)Nzna = (3,4, 5,6) y = [3, 4, 5] z = {'a': 5, 'b':15, 'c':True} x = len(y) + 5 - a[ 3] - a[2] + len(z) - z[ 'b'] rmr=r$r$r%test_continuation1szTestRoundtrip.test_continuationcCs|d|ddS)Nz)x=1+\ 1 # This is a comment\ # This also z# Comment \ x = 0rYr=r$r$r%rG>s z)TestRoundtrip.test_backslash_continuationcCs|ddS)Nz'' ''rYr=r$r$r%test_string_concatenationGsz'TestRoundtrip.test_string_concatenationc Csddl}ddl}td}tj|p*tj}|tj| |d}| tj|ddD]}| tj|d|q^t ds| |d}|D]z}tj d krtd |t|d F}|j|d ||Wdn1s0YWdq1s0YqdS) NrrUztest*.pyztest_unicode_identifiers.py)bufferZbuiltinZfileioinspectrgplatformsysz test_%s.pyZcpu rrrfr)globrandomrrVrgrhrjcurdirriescaperemoveZis_resource_enabledsampleverboserrZsubTestrm)r.rarbrWtempdirZ testfilesr1Ztestfiler$r$r%test_random_filesKs      zTestRoundtrip.test_random_filescCs,t|tr|d}ttt|jdS)Nr()rPrr)rrrr*r\)r.rTr$r$r% roundtriphs  zTestRoundtrip.roundtripcCs6d}||d}||d|d||dS)Nzif False: x=3 x=3 rrr)rjsplitr+rm)r.rTZ codelinesr$r$r%#test_indentation_semantics_retainedmsz1TestRoundtrip.test_indentation_semantics_retainedN) rRrSrTrmrXrZrGr[rirjrlr$r$r$r%rls '  rl__main__)*testrrrrrrrr r r r r rrrrriorrZunittestrrZtest.test_grammarrrrgr r&r'rUrarbrerzrrr;rlrRmainr$r$r$r%s> D %&)