a ze%@sNddlmZddlmZmZmZmZmZmZmZm Z m Z m Z m Z m ZmZmZmZddlmZmZddlZddlmZmZddlmZmZddlZddlZddZGd d d eZGd d d eZd dZ GdddeZ!GdddeZ"GdddeZ#GdddeZ$GdddeZ%GdddeZ&GdddeZ'e(dkrJe)dS))support)tokenize _tokenize untokenizeNUMBERNAMEOPSTRING ENDMARKERENCODINGtok_namedetect_encodingopen Untokenizergenerate_tokensNEWLINE)BytesIOStringION)TestCasemock)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALSc Csg}t|}|ddv}|D]b\}}}}} |tkr:q|rT|tkrT|d|krTq t|}|d|dd|dd|d|q |S)Nz r Z10 Z13)len splitlinesr rr append) Ztoken_generatorZ source_stringresultZ num_linesZmissing_trailing_nltypetokenstartendliner$F/opt/bitninja-python-dojo/embedded/lib/python3.9/test/test_tokenize.pystringify_tokens_from_sources  (r&c@seZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)S)* TokenizeTestcCs<t|d}tt|j|}||dg|dS)Nutf-8z* ENCODING 'utf-8' (0, 0) (0, 0))rencoder&rreadline assertEqualrstriprselfsexpectedfrr$r$r%check_tokenize*s zTokenizeTest.check_tokenizecCsDtdd}tt|j}||djt||djtdS)Nxr(r) rr)listrr*r+rrr )r.r1tokensr$r$r%test_implicit_newline4sz"TokenizeTest.test_implicit_newlinecCsd|dd|ddd}t|j}|tdt|D]}q ~0x15z NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) OP '~' (1, 10) (1, 11) NUMBER '0x15' (1, 11) (1, 15) z2134568 != 1231515z NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '1231515' (1, 11) (1, 18) z(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '200000000' (1, 14) (1, 23) z0xdeadbeef != -1z NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) z0xdeadc0de & 12345z NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '12345' (1, 13) (1, 18) z0xFF & 0x15 | 1234z NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) r2r.r$r$r%test_intas       zTokenizeTest.test_intcCs4|dd|dd|dd|dddS) Nzx = 0z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0' (1, 4) (1, 5) zx = 0xfffffffffffz NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xfffffffffff' (1, 4) (1, 17) zx = 123141242151251616110z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151251616110' (1, 4) (1, 25) zx = -15921590215012591z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '15921590215012591' (1, 5) (1, 22) r<r=r$r$r% test_longs   zTokenizeTest.test_longcCsX|dd|dd|dd|dd|d d |d d |d ddS)Nz x = 3.14159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) z x = 314159.z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) z x = .314159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) z x = 3e14159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) z x = 3E123z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) z x+y = 3e-1230z NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) z x = 3.14e159z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) r<r=r$r$r% test_floats      zTokenizeTest.test_floatcCsJdd}tD]}d|vrq ||||q tD]}||||q0dS)NcSs<t|d}t|jD]\}}}}}|tkr|SqdS)Nr(z invalid token)rr)rr*r)r/r1Ztoktyper r!r"r#r$r$r% number_tokens  z;TokenizeTest.test_underscore_literals..number_token()rr+rZassertNotEqual)r.rAZlitr$r$r%test_underscore_literalssz%TokenizeTest.test_underscore_literalscCs<|dd|dd|dd|dd|d d |d d |d d|dd|dd|dd|dd|dd|dd|dd|dd|dd |d!d"|d#d$|d%d&|d'd(|d)d*|d+d,|d-d.|d/d0|d1d2|d3d4dS)5Nzx = ''; y = ""a5 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "''" (1, 4) (1, 6) OP ';' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '=' (1, 10) (1, 11) STRING '""' (1, 12) (1, 14) zx = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '\'"\'' (1, 4) (1, 7) OP ';' (1, 7) (1, 8) NAME 'y' (1, 9) (1, 10) OP '=' (1, 11) (1, 12) STRING '"\'"' (1, 13) (1, 16) zx = "doesn't "shrink", does it"z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"doesn\'t "' (1, 4) (1, 14) NAME 'shrink' (1, 14) (1, 20) STRING '", does it"' (1, 20) (1, 31) zx = 'abc' + 'ABC'z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "'abc'" (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING "'ABC'" (1, 12) (1, 17) zy = "ABC" + "ABC"z NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"ABC"' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING '"ABC"' (1, 12) (1, 17) z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "r'abc'" (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING "r'ABC'" (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING "R'ABC'" (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING "R'ABC'" (1, 31) (1, 37) z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'r"abc"' (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING 'r"ABC"' (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING 'R"ABC"' (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING 'R"ABC"' (1, 31) (1, 37) zu'abc' + U'abc'z STRING "u'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "U'abc'" (1, 9) (1, 15) zu"abc" + U"abc"z STRING 'u"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'U"abc"' (1, 9) (1, 15) zb'abc' + B'abc'z STRING "b'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "B'abc'" (1, 9) (1, 15) zb"abc" + B"abc"z STRING 'b"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'B"abc"' (1, 9) (1, 15) z%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "bR'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Br'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "BR'abc'" (1, 30) (1, 37) z%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'bR"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Br"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'BR"abc"' (1, 30) (1, 37) z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "rB'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Rb'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "RB'abc'" (1, 30) (1, 37) z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'rB"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Rb"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) z "a\ de\ fg"z3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3) zu"a\ de"z/ STRING 'u"a\\\nde"' (1, 0) (2, 3) zrb"a\ d"z/ STRING 'rb"a\\\nd"' (1, 0) (2, 2) z """a\ b"""z0 STRING '"""a\\\nb"""' (1, 0) (2, 4) z u"""a\ b"""z1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4) zrb"""a\ b\ c"""z7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4) zf"abc"z/ STRING 'f"abc"' (1, 0) (1, 6) z fR"a{b}c"z/ STRING 'fR"a{b}c"' (1, 0) (1, 9) z f"""abc"""z0 STRING 'f"""abc"""' (1, 0) (1, 10) z f"abc\ def"z1 STRING 'f"abc\\\ndef"' (1, 0) (2, 4) z Rf"abc\ def"z2 STRING 'Rf"abc\\\ndef"' (1, 0) (2, 4) r<r=r$r$r% test_stringsT           zTokenizeTest.test_stringcCs(|dd|dd|dddS)Nz!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) zdef d01v_(a=1, *k, **w): passa NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) z&def d23(a: str, b: int=3) -> int: passa NAME 'def' (1, 0) (1, 3) NAME 'd23' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ':' (1, 9) (1, 10) NAME 'str' (1, 11) (1, 14) OP ',' (1, 14) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'int' (1, 19) (1, 22) OP '=' (1, 22) (1, 23) NUMBER '3' (1, 23) (1, 24) OP ')' (1, 24) (1, 25) OP '->' (1, 26) (1, 28) NAME 'int' (1, 29) (1, 32) OP ':' (1, 32) (1, 33) NAME 'pass' (1, 34) (1, 38) r<r=r$r$r% test_functions  zTokenizeTest.test_functioncCs|dddS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa NAME 'if' (1, 0) (1, 2) NUMBER '1' (1, 3) (1, 4) OP '<' (1, 5) (1, 6) NUMBER '1' (1, 7) (1, 8) OP '>' (1, 9) (1, 10) NUMBER '1' (1, 11) (1, 12) OP '==' (1, 13) (1, 15) NUMBER '1' (1, 16) (1, 17) OP '>=' (1, 18) (1, 20) NUMBER '5' (1, 21) (1, 22) OP '<=' (1, 23) (1, 25) NUMBER '0x15' (1, 26) (1, 30) OP '<=' (1, 31) (1, 33) NUMBER '0x12' (1, 34) (1, 38) OP '!=' (1, 39) (1, 41) NUMBER '1' (1, 42) (1, 43) NAME 'and' (1, 44) (1, 47) NUMBER '5' (1, 48) (1, 49) NAME 'in' (1, 50) (1, 52) NUMBER '1' (1, 53) (1, 54) NAME 'not' (1, 55) (1, 58) NAME 'in' (1, 59) (1, 61) NUMBER '1' (1, 62) (1, 63) NAME 'is' (1, 64) (1, 66) NUMBER '1' (1, 67) (1, 68) NAME 'or' (1, 69) (1, 71) NUMBER '5' (1, 72) (1, 73) NAME 'is' (1, 74) (1, 76) NAME 'not' (1, 77) (1, 80) NUMBER '1' (1, 81) (1, 82) OP ':' (1, 82) (1, 83) NAME 'pass' (1, 84) (1, 88) r<r=r$r$r%test_comparisonszTokenizeTest.test_comparisoncCs|dddS)Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) r<r=r$r$r% test_shiftszTokenizeTest.test_shiftcCs|dddS)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '1' (1, 17) (1, 18) OP '+' (1, 19) (1, 20) NUMBER '0x124' (1, 21) (1, 26) OP '+' (1, 27) (1, 28) NAME 'z' (1, 29) (1, 30) OP '+' (1, 31) (1, 32) NAME 'a' (1, 33) (1, 34) OP '[' (1, 34) (1, 35) NUMBER '5' (1, 35) (1, 36) OP ']' (1, 36) (1, 37) r<r=r$r$r% test_additiveszTokenizeTest.test_additivecCs|dddS)Nzx = 1//1*1/5*12%0x12@42a NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) OP '@' (1, 20) (1, 21) NUMBER '42' (1, 21) (1, 23) r<r=r$r$r%test_multiplicativesz TokenizeTest.test_multiplicativecCs|dd|dddS)Nz~1 ^ 1 & 1 |1 ^ -1a OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) z-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) r<r=r$r$r% test_unary(s zTokenizeTest.test_unarycCs|dddS)Nz/import sys, time x = sys.modules['time'].time()a NAME 'import' (1, 0) (1, 6) NAME 'sys' (1, 7) (1, 10) OP ',' (1, 10) (1, 11) NAME 'time' (1, 12) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'x' (2, 0) (2, 1) OP '=' (2, 2) (2, 3) NAME 'sys' (2, 4) (2, 7) OP '.' (2, 7) (2, 8) NAME 'modules' (2, 8) (2, 15) OP '[' (2, 15) (2, 16) STRING "'time'" (2, 16) (2, 22) OP ']' (2, 22) (2, 23) OP '.' (2, 23) (2, 24) NAME 'time' (2, 24) (2, 28) OP '(' (2, 28) (2, 29) OP ')' (2, 29) (2, 30) r<r=r$r$r% test_selectorMszTokenizeTest.test_selectorcCs|dddS)Nz @staticmethod def foo(x,y): passa OP '@' (1, 0) (1, 1) NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) OP '(' (2, 7) (2, 8) NAME 'x' (2, 8) (2, 9) OP ',' (2, 9) (2, 10) NAME 'y' (2, 10) (2, 11) OP ')' (2, 11) (2, 12) OP ':' (2, 12) (2, 13) NAME 'pass' (2, 14) (2, 18) r<r=r$r$r% test_methodcszTokenizeTest.test_methodcCs|dddS)Nzdef f(): if x passa_ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) INDENT '\t' (2, 0) (2, 1) NAME 'if' (2, 1) (2, 3) NAME 'x' (2, 4) (2, 5) NEWLINE '\n' (2, 5) (2, 6) INDENT ' \t' (3, 0) (3, 9) NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) r<r=r$r$r% test_tabstszTokenizeTest.test_tabscCs|dddS)Nu!Örter = 'places' grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "'places'" (1, 8) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "'green'" (2, 7) (2, 14) r<r=r$r$r%test_non_ascii_identifierssz'TokenizeTest.test_non_ascii_identifierscCs|dddS)Nu#Örter = u'places' grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "u'places'" (1, 8) (1, 17) NEWLINE '\n' (1, 17) (1, 18) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "U'green'" (2, 7) (2, 15) r<r=r$r$r% test_unicodeszTokenizeTest.test_unicodecCs|dd|dd|dd|dd|d d |d d |d d|dd|dd|dd|dd|dd|dd|dd|dd|dd |d!d"|d#d$|d%d&|d'd(|d)d*dS)+Nz async = 1z NAME 'async' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) za = (async = 1)a8 NAME 'a' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '(' (1, 4) (1, 5) NAME 'async' (1, 5) (1, 10) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) zasync()z NAME 'async' (1, 0) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) zclass async(Bar):passa< NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP '(' (1, 11) (1, 12) NAME 'Bar' (1, 12) (1, 15) OP ')' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 17) (1, 21) zclass async:passz NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP ':' (1, 11) (1, 12) NAME 'pass' (1, 12) (1, 16) z await = 1z NAME 'await' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) z foo.asyncz NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) zasync for a in b: passa; NAME 'async' (1, 0) (1, 5) NAME 'for' (1, 6) (1, 9) NAME 'a' (1, 10) (1, 11) NAME 'in' (1, 12) (1, 14) NAME 'b' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 18) (1, 22) zasync with a as b: passa< NAME 'async' (1, 0) (1, 5) NAME 'with' (1, 6) (1, 10) NAME 'a' (1, 11) (1, 12) NAME 'as' (1, 13) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'pass' (1, 19) (1, 23) z async.fooz NAME 'async' (1, 0) (1, 5) OP '.' (1, 5) (1, 6) NAME 'foo' (1, 6) (1, 9) asyncz/ NAME 'async' (1, 0) (1, 5) zasync #comment awaitz NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) COMMENT '#comment' (2, 0) (2, 8) NL '\n' (2, 8) (2, 9) NAME 'await' (3, 0) (3, 5) zasync ... awaitz NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) OP '...' (2, 0) (2, 3) NEWLINE '\n' (2, 3) (2, 4) NAME 'await' (3, 0) (3, 5) z async awaitz NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) NAME 'await' (2, 0) (2, 5) z foo.async + 1z NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '1' (1, 12) (1, 13) zasync def foo(): passa; NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NAME 'pass' (1, 17) (1, 21) zNasync def foo(): def foo(await): await = 1 if 1: await async += 1 a NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'def' (2, 2) (2, 5) NAME 'foo' (2, 6) (2, 9) OP '(' (2, 9) (2, 10) NAME 'await' (2, 10) (2, 15) OP ')' (2, 15) (2, 16) OP ':' (2, 16) (2, 17) NEWLINE '\n' (2, 17) (2, 18) INDENT ' ' (3, 0) (3, 4) NAME 'await' (3, 4) (3, 9) OP '=' (3, 10) (3, 11) NUMBER '1' (3, 12) (3, 13) NEWLINE '\n' (3, 13) (3, 14) DEDENT '' (4, 2) (4, 2) NAME 'if' (4, 2) (4, 4) NUMBER '1' (4, 5) (4, 6) OP ':' (4, 6) (4, 7) NEWLINE '\n' (4, 7) (4, 8) INDENT ' ' (5, 0) (5, 4) NAME 'await' (5, 4) (5, 9) NEWLINE '\n' (5, 9) (5, 10) DEDENT '' (6, 0) (6, 0) DEDENT '' (6, 0) (6, 0) NAME 'async' (6, 0) (6, 5) OP '+=' (6, 6) (6, 8) NUMBER '1' (6, 9) (6, 10) NEWLINE '\n' (6, 10) (6, 11) z)async def foo(): async for i in 1: passa NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'async' (2, 2) (2, 7) NAME 'for' (2, 8) (2, 11) NAME 'i' (2, 12) (2, 13) NAME 'in' (2, 14) (2, 16) NUMBER '1' (2, 17) (2, 18) OP ':' (2, 18) (2, 19) NAME 'pass' (2, 20) (2, 24) DEDENT '' (3, 0) (3, 0) zasync def foo(async): awaitah NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) NAME 'async' (1, 14) (1, 19) OP ')' (1, 19) (1, 20) OP ':' (1, 20) (1, 21) NAME 'await' (1, 22) (1, 27) z@def f(): def baz(): pass async def bar(): pass await = 2a NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) zFasync def f(): def baz(): pass async def bar(): pass await = 2a NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'f' (1, 10) (1, 11) OP '(' (1, 11) (1, 12) OP ')' (1, 12) (1, 13) OP ':' (1, 13) (1, 14) NEWLINE '\n' (1, 14) (1, 15) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) r<r=r$r$r% test_asyncs:            , %zTokenizeTest.test_asyncN)__name__ __module__ __qualname__r2r7r;r>r?r@rCrDrErFrGrHrIrJrKrLrMrNrOrQr$r$r$r%r'#s* %9('<& %  r'c@seZdZddZdS)GenerateTokensTestcCs0t|}tt|j|}|||dSN)rr&rr*r+r,rr-r$r$r%r2sz!GenerateTokensTest.check_tokenizeN)rRrSrTr2r$r$r$r%rUsrUcCsg}tt|dj}|D]T\}}}}}|tkrbd|vrb|tdftdftt |ftdfgq| ||fqt | dS)Nr(.DecimalrB)) rrr)r*rextendrrr reprrrdecode)r/rgtoknumtokval_r$r$r%decistmts rac@seZdZddZdS)TestMisccCsPddlm}d}|t|d|tt|d|tt||ddS)Nr)rXz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)decimalrXr+raZ assertRegexr[eval)r.rXr/r$r$r% test_decistmts  zTestMisc.test_decistmtN)rRrSrTrer$r$r$r%rbsrbc@s@eZdZdZddZddZddZdd Zd d Zd d Z dS)TestTokenizerAdheresToPep0263zU Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263. cCs,tjtjt|}t|t|ddS)Nrb)ospathjoindirname__file__ TestRoundtripcheck_roundtripr)r.filenamerir$r$r% _testFilesz'TestTokenizerAdheresToPep0263._testFilecCsd}||dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtrpr.r1r$r$r%'test_utf8_coding_cookie_and_no_utf8_bomszETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomcCsd}|t|j|dS)a As per PEP 0263, if a file starts with a utf-8 BOM signature, the only allowed encoding for the comment is 'utf-8'. The text file used in this test starts with a BOM signature, but specifies latin1 as the coding, so verify that a SyntaxError is raised, which matches the behaviour of the interpreter when it encounters a similar condition. z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN assertRaises SyntaxErrorrprrr$r$r%&test_latin1_coding_cookie_and_utf8_bomszDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomcCsd}||dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtrqrrr$r$r%"test_no_coding_cookie_and_utf8_bomsz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomcCsd}||dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtrqrrr$r$r%$test_utf8_coding_cookie_and_utf8_bomszBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomcCs$|t|jd|t|jddS)Nz bad_coding.pyzbad_coding2.pyrtr=r$r$r%test_bad_coding_cookiesz4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN) rRrSrT__doc__rprsrwrxryrzr$r$r$r%rfs rfc@seZdZddZddZdS) Test_TokenizecsPd}|ddfdd}tt|dddd}d g}|||d dS) N "ЉЊЈЁЂ"r(Fcss dSdSdSNTr$r$firstr#r$r%r* szNTest_Tokenize.test__tokenize_decodes_with_specified_encoding..readlineencodingr4r})rr)rr}zbytes not decoded with encoding)r)r5rr+)r.literalr*r6expected_tokensr$rr%.test__tokenize_decodes_with_specified_encoding s  z.readlinerr4rz*string not tokenized when encoding is None)r5rr+)r.r*r6rr$rr%1test__tokenize_does_not_decode_with_encoding_nones z?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rRrSrTrrr$r$r$r%r|sr|c@seZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)S)*TestDetectEncodingcsdfdd}|S)Nrcs$tkrt}d7|SNr)r StopIteration)r#indexlinesr$r%r*1s  z1TestDetectEncoding.get_readline..readliner$r.rr*r$rr% get_readline/szTestDetectEncoding.get_readlinecCs>d}t||\}}||d||t|dddS)N) # something print(something) do_something(else) r()r rr+r5r.rrconsumed_linesr$r$r%test_no_bom_no_encoding_cookie:s z1TestDetectEncoding.test_no_bom_no_encoding_cookiecCs6d}t||\}}||d||ddgdS)N)s# something rr utf-8-sigrrr rr+rr$r$r%test_bom_no_cookieDs  z%TestDetectEncoding.test_bom_no_cookiecCs4d}t||\}}||d||dgdS)N)# -*- coding: latin-1 -*- rr iso-8859-1rrrr$r$r%test_cookie_first_line_no_bomOs z0TestDetectEncoding.test_cookie_first_line_no_bomcCs4d}t||\}}||d||dgdS)N)s# coding=utf-8 rrrs# coding=utf-8 rrr$r$r%&test_matched_bom_and_cookie_first_lineYs z9TestDetectEncoding.test_matched_bom_and_cookie_first_linecCs d}||}|tt|dS)N)s## vim: set fileencoding=ascii : rrrrurvr rr$r$r%.BunkcSs||_||_d|_dS)Nr)nameZ_lines_index)r.rrir$r$r%__init__szDTestDetectEncoding.test_filename_in_exception..Bunk.__init__cs.|jtkrt|j}|jd7_|Sr)rrr)r.r#rr$r%r*s  zDTestDetectEncoding.test_filename_in_exception..Bunk.readlineN)rRrSrTrr*r$rr$r%Bunk srz.*{})rurvrr r*r8format)r.rirZinsr$rr%test_filename_in_exceptions  ( z-TestDetectEncoding.test_filename_in_exceptioncCsTtd}tjd|d|ttdWdn1s:0Y||jdS)Ns #coding:xxxztokenize._builtin_open)Z return_valueZfoobar)rrZpatchrurvrZ assertTrueclosed)r.mr$r$r%test_open_error#s,z"TestDetectEncoding.test_open_errorN)rRrSrTrrrrrrrrrrrrrrrrrrrrr$r$r$r%r-s(              rc@s<eZdZddZddZddZddZd d Zd d Zd S) TestTokenizecsddl}tdfdd}fdd}dfdd}|j}|j}||_||_z,t|}|t|gdW||_||_n||_||_0|dS) Nrcs ddgfS)Nfirstsecondr$)r*rr$r%mock_detect_encoding1sz8TestTokenize.test_tokenize..mock_detect_encodingcs"|g}|}|r||q|SrV)r)r*routZ next_line) encoding_usedr$r%mock__tokenize4s z2TestTokenize.test_tokenize..mock__tokenizecs d7dkrdStS)Nrr)strr)r$)counterr$r% mock_readline@sz1TestTokenize.test_tokenize..mock_readline)rr1234)robjectr rr+r5)r.Ztokenize_modulerrrZorig_detect_encodingZorig__tokenizeresultsr$)rrrr% test_tokenize-s*    zTestTokenize.test_tokenizecCsfg}tdD]}|dj|dq |dd|}ttt|dj}| |dj ddS)Nizdef i{i}(): return {i})iZOK r() rangerrrjr5rrr)r*r+string)r.bufrZtoksr$r$r%test_oneline_defsUs   zTestTokenize.test_oneline_defscGsttt|dj}t|}|t|d||t|djtt t |D]&}|t||djt||qV|t|d|jtt j |t|d|jtt j dS)Nr(rrrr)r5rrr)r*rr+r exact_typer rr rr )r.ZopstrZoptypesr6Z num_optypesrr$r$r%assertExactTypeEqualas   z!TestTokenize.assertExactTypeEqualcCs|dtjtj|dtjtj|dtj|dtj|dtj|dtj |dtj |dtj |d tj |d tj |d tj|d tj|d tj|dtj|dtj|dtj|dtjtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj|dtj |dtj!|dtj"|dtj#|d tj$|d!tj%|d"tj&|d"tj&|d#tj'|d$tj(|d%tj)|d&tj*|d'tj+|d(tj,|d)tj-|d*tj.|d+tj/|d,tj0|d-t1tjt2tj t1tjt2tjt1tjt2 |d.tjtj2tjtj2tjtj2tj|d/tjtjtj1tjtj2tjdS)0Nz()z[]:,;+r*/|&<>=rW%z{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z {1, 2, 3}z ^(x & 0x1))3rr LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE CIRCUMFLEX LEFTSHIFT RIGHTSHIFT DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL SLASHEQUAL PERCENTEQUAL AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUAL COLONEQUALELLIPSISRARROWATATEQUALrrr=r$r$r%test_exact_typeos zTestTokenize.test_exact_typecCs|dtjdS)Nz @ )rr r4r=r$r$r%%test_pathological_trailing_whitespacesz2TestTokenize.test_pathological_trailing_whitespacecCsd}tjtjtjtjtjtjg}ttt | dj }| t |djt ttdD]&}| t ||djt ||qZ| t |djt tjdS)Nz b = 1 #testr(rrr)r rrrrNLCOMMENTr5rrr)r*r+r rr rr )r.sourcerr6rr$r$r%5test_comment_at_the_end_of_the_source_without_newlines $zBTestTokenize.test_comment_at_the_end_of_the_source_without_newlineN) rRrSrTrrrr6r7r<r$r$r$r%r+s ( Arc@s$eZdZddZddZddZdS)UntokenizeTestcCsnt}d|_d|_|t}|dWdn1s<0Y||jjdd|t|jddS)Nr)rrrz'start (1,3) precedes previous end (2,2))rr) rprev_rowprev_colru ValueErroradd_whitespacer+ exceptionargs)r.ucmr$r$r%test_bad_input_orders (z#UntokenizeTest.test_bad_input_ordercCsdt}d|_d|_g|_|d||jdgd|_|d||jgdt|ddS)Nr)rr\ r)rH)rGz\ \ rza b c \ c )rr>r?r6rAr+rmrn)r.rDr$r$r%test_backslash_continuations  z*UntokenizeTest.test_backslash_continuationcCst}tdf}tdf|g}||tg||jdgt}||t|gdt}||t|d||jd|tt|ddS)NZHellor(zHello sHello ) rrr compatiterr+r6rr)r.rDr r6r$r$r%test_iter_compats zUntokenizeTest.test_iter_compatN)rRrSrTrFrIrLr$r$r$r%r=s  r=c@sLeZdZddZddZddZddZd d Zd d Zd dZ ddZ dS)rmc Cst|tr|d}n|}|t|jddj}tt |}dd|D}t |}t|jddj}ddt |D}| ||t |} t| jddj} ddt | D} | | |dS) a) Test roundtrip for `untokenize`. `f` is an open file or a string. The source code in f is tokenized to both 5- and 2-tuples. Both sequences are converted back to source code via tokenize.untokenize(), and the latter tokenized again to 2-tuples. The test fails if the 3 pair tokenizations do not match. When untokenize bugs are fixed, untokenize with 5-tuples should reproduce code that does not contain a backslash continuation following spaces. A proper test should test this. r(T)keependscSsg|]}|ddqSNrr$.0r:r$r$r% rz1TestRoundtrip.check_roundtrip..cSsg|]}|ddqSrNr$rOr$r$r%rQrcSsg|]}|ddqSrNr$rOr$r$r%rQrN) isinstancerr)readcloserKr__next__r5rrr+) r.r1coder*Ztokens5Ztokens2Z bytes_from2Z readline2Z tokens2_from2Z bytes_from5Z readline5Z tokens2_from5r$r$r%rns    zTestRoundtrip.check_roundtripcCs|d|d|dtd}t|d}||Wdn1sR0Y|d|d|d|d dS) Nzif x == 1: print(x) z # This is a comment # This also zif x == 1 : print(x) tokenize_tests.txtrgzoif x == 1: # A comment by itself. print(x) # Comment here, too. # Another comment. after_if = True zPif (x # The comments need to go in the right place == 1): print('x==1') zclass Test: # A comment here # A comment with weird indent after_com = 5 def x(m): return m*5 # a one liner def y(m): # A whitespace after the colon return y*4 # 3-space indent zttry: import somemodule except ImportError: # comment print('Can not import' # comment2 )else: print('Loaded') )rnrfindfiler)r.fnr1r$r$r%test_roundtrip s     (   zTestRoundtrip.test_roundtripcCs|ddS)Nzna = (3,4, 5,6) y = [3, 4, 5] z = {'a': 5, 'b':15, 'c':True} x = len(y) + 5 - a[ 3] - a[2] + len(z) - z[ 'b'] rnr=r$r$r%test_continuation1szTestRoundtrip.test_continuationcCs|d|ddS)Nz)x=1+\ 1 # This is a comment\ # This also z# Comment \ x = 0r[r=r$r$r%rI>s z)TestRoundtrip.test_backslash_continuationcCs|ddS)Nz'' ''r[r=r$r$r%test_string_concatenationGsz'TestRoundtrip.test_string_concatenationc Csddl}ddl}td}tj|p*tj}|tj| |d}| tj|ddD]}| tj|d|q^t ds| |d}|D]z}tj d krtd |t|d F}|j|d ||Wdn1s0YWdq1s0YqdS) NrrWztest*.pyztest_unicode_identifiers.py)bufferZbuiltinZfileioinspectrhplatformsysz test_%s.pyZcpu rrrgr)globrandomrrXrhrirkcurdirrjescaperemoveZis_resource_enabledsampleverboserrZsubTestrn)r.rcrdrYtempdirZ testfilesr1Ztestfiler$r$r%test_random_filesKs      zTestRoundtrip.test_random_filescCs,t|tr|d}ttt|jdS)Nr()rRrr)rrrr*r\)r.rVr$r$r% roundtriphs  zTestRoundtrip.roundtripcCs6d}||d}||d|d||dS)z Ensure that although whitespace might be mutated in a roundtrip, the semantic meaning of the indentation remains consistent. zif False: x=3 x=3 rrrN)rlsplitr+rn)r.rVZ codelinesr$r$r%#test_indentation_semantics_retainedmsz1TestRoundtrip.test_indentation_semantics_retainedN) rRrSrTrnrZr\rIr]rkrlrnr$r$r$r%rms '  rm__main__)*testrrrrrrrr r r r r rrrrriorrZunittestrrZtest.test_grammarrrrhr r&r'rUrarbrfr|rrr=rmrRmainr$r$r$r%s> D %&)