a  zer@sddlZddlZddlZddlmZddlmZmZddlm Z m Z m Z m Z m Z ededhddlmZddlmZmZmZddlmZmZmZdd lmZdd lmZdd lm Z Wdn1s0YGd d d ej!Z"GdddZ#Gdddej!Z$dS)N) test_tools)DictAny) TokenInfoNAMENEWLINENUMBEROPZ peg_generator)GeneratedParser) parse_stringgenerate_parser make_parser)GrammarVisitor GrammarErrorGrammar)ASTGrammarPrinter)Parser)PythonParserGeneratorc@seZdZddddZddddZddddZddd d Zddd d Zddd dZddddZ ddddZ ddddZ ddddZ ddddZ ddddZddddZddddZdddd Zddd!d"Zddd#d$Zddd%d&Zddd'd(Zddd)d*Zddd+d,Zddd-d.Zddd/d0Zddd1d2Zddd3d4Zddd5d6Zddd7d8Zddd9d:Zddd;d<Zddd=d>Z ddd?d@Z!dS)A TestPegenNreturncCsvd}d}t|t}|j}|t|t||t|dd|t|ddd}|t|d|dS) Nzl start: sum NEWLINE sum: t1=term '+' t2=term { action } | term term: NUMBER [ start: sum NEWLINE sum: term '+' term | term term: NUMBER startstart: sum NEWLINEsumsum: term '+' term | termzERule('term', None, Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))term) r GrammarParserrules assertEqualstrtextwrapdedentstriprepr)selfgrammar_sourceexpectedgrammarrZ expected_reprr)V/opt/bitninja-python-dojo/embedded/lib/python3.9/test/test_peg_generator/test_pegen.pytest_parse_grammars zTestPegen.test_parse_grammarcCs6d}d}t|t}|t|jdt|dS)Nzt start: zero | one | one zero | one one | one zero zero | one zero one | one one zero | one one one z start: | zero | one | one zero | one one | one zero zero | one zero one | one one zero | one one one r)r rrr rr!r"r#)r%r&r'r(r)r)r*test_long_rule_str,s zTestPegen.test_long_rule_strcCsPd}t|tj}|t|dd|t|dd|t|dddS)Nz{ start[int]: sum NEWLINE sum[int]: t1=term '+' t2=term { action } | term term[int]: NUMBER rrrrrzFRule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])])))r rrrr r$)r%r(rr)r)r*test_typed_rules>s  zTestPegen.test_typed_rulesc Csd}t|tj}|t|dd|t|dd|t|ddt|}td|}|t t dd d dd ggt t d d d dd gksJtd|}|t t dd ddd gt t dd ddd ggt t d dddd gksJdS)NA start: ','.thing+ NEWLINE thing: NUMBER rzstart: ','.thing+ NEWLINEzXRule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf("','"), NameLeaf('thing'thingz thing: NUMBER42 42rr3stringrendline r3z1, 2 1r3r32r3r3) r rrrr assertTruer$ startswithr rrr)r%r(r parser_classnoder)r)r* test_gatherMs(    zTestPegen.test_gatherc CsJd}t|}td|}||ttdddddggttdddddgdS) Nrr0r1r2r4r6r:r;r r rrrrr%r(rFrGr)r)r*test_expr_grammargs zTestPegen.test_expr_grammarc Csd}t|}td|}||ttdddddgttdddddttd dd ddgggttd d d ddgtd |}||ttdddd dgdgttd ddd dgdS)NzW start: sum NEWLINE sum: term ('+' term)? term: NUMBER z1+2 r=r2r>r6+r4r?r;r:r@1 r r rrrr rrJr)r)r*test_optional_operatorts   z TestPegen.test_optional_operatorc Csd}t|}td|}||ttdddddgttdddddgttd dd ddgtd |}||ttdddd dgdgttd ddd dgdS) NzQ start: sum NEWLINE sum: term '+' ? term: NUMBER z1+ r=r2r>r6rLr4r:r;rMrNrJr)r)r*test_optional_literals  zTestPegen.test_optional_literalc Csd}t|}td|}||ttdddddgttddd ddttd d d ddgggttd d dddgtd|}||ttdddddgdgttd ddddgdS)NzV start: sum NEWLINE sum: term ['+' term] term: NUMBER z1 + 2 r=r2r>r6rLr4r;r?r@rBr:r3rMrNrJr)r)r*test_alt_optional_operators   z$TestPegen.test_alt_optional_operatorc Csd}t|}td|}||ttdddddgttddd ddggttd d d ddgggttd d dddgtd|}||ttdddddggttd ddddgdS)NzC start: thing thing* NEWLINE thing: NUMBER 1 2 3 r=r2r>r6r?r4r;3r@rBr:rQrMrIrJr)r)r*test_repeat_0_simples   zTestPegen.test_repeat_0_simplec Csd}t|}td|}||ttdddddgttddd ddttd d d ddgggttdd dddttdddddggggttdddddgdS)NzF start: term ('+' term)* NEWLINE term: NUMBER 1 + 2 + 3 r=r2r>r6rLr4r;r?r@rBrQr3rUr3r3 r:r3 rNrJr)r)r*test_repeat_0_complexs  zTestPegen.test_repeat_0_complexc Csd}t|}td|}||ttdddddgttddd ddggttd d d ddgggttd d dddg|ttd|Wdn1s0YdS)NzC start: thing thing+ NEWLINE thing: NUMBER rTr=r2r>r6r?r4r;rUr@rBr:rQrM)r r rrrr assertRaises SyntaxErrorrJr)r)r*test_repeat_1_simples  zTestPegen.test_repeat_1_simplec Csd}t|}td|}||ttdddddgttddd ddttd d d ddgggttdd dddttdddddggggttdddddg|ttd|Wdn1s0YdS)NzF start: term ('+' term)+ NEWLINE term: NUMBER rWr=r2r>r6rLr4r;r?r@rBrQrXrUrZr\r:r^rM) r r rrrr rrarbrJr)r)r*test_repeat_1_complexs$  zTestPegen.test_repeat_1_complexc Csnd}t|}td|}||ttdddddgttddd ddgttd d d ddggttd d dddgdS)Nr.z1, 2, 3 r=r2r>r6r?r;r@rUrQrXr:rZrIrJr)r)r*test_repeat_with_sep_simples z%TestPegen.test_repeat_with_sep_simplec Csd}t|t}t|}|j}||dj||dj||dj||dj||dj||djtd|}||tt d d d dd ggtt d dddd tt ddddd ggtt d dddd tt ddddd ggtt ddddd gdS)Nz start: expr NEWLINE expr: ('-' term | expr '+' term | term) term: NUMBER foo: NAME+ bar: NAME* baz: NAME? rexprrfoobarZbazrWr=r2r>r6rLr4r;r?r@rBrQrXrUrZr\r:r^) r rr r assertFalseleft_recursiverDrrrr r)r%r&r(rFrrGr)r)r*test_left_recursive's*   zTestPegen.test_left_recursivecCs:d}t|}td|}t|dd}t|}||ddS)Na start: expr NEWLINE? $ { ast.Expression(expr, lineno=1, col_offset=0) } expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } | expr '-' term { ast.BinOp(expr, ast.Sub(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } | term { term } ) term: ( l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } | factor { factor } ) factor: ( '(' expr ')' { expr } | atom { atom } ) atom: ( n=NAME { ast.Name(id=n.string, ctx=ast.Load(), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } | n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } ) z(1 + 2*3 + 5)/(6 - 2) evalg@)r r compilermr)r%r(rFrGcodevalr)r)r*test_python_exprGs   zTestPegen.test_python_exprcCsJd}t|t}t}t||}|j}||dj||djdS)Nz> start: sign NUMBER sign: ['-' | '+'] rsign) r rioStringIOrrrinullablerDr%r&r(outgenrrr)r)r* test_nullable_s  zTestPegen.test_nullablecCsjd}t|t}t}t||}|j}||dj||dj||dj ||dj dS)Nz@ start: NUMBER | sign start sign: ['-'] rrr) r rrsrtrrrirurDrjrvr)r)r*test_advanced_left_recursiveks  z&TestPegen.test_advanced_left_recursivec CsRd}t|t}t}t||}|j}||dj||dj||dj| di}t | ||d}td|}| |t tdd d dd gt td d ddd gt tddddd gt td dddd gt tddddd gtd|}||| |t tdd d dd gt tdd ddd gt td dddd gt tddddd gdS)NzV start: foo 'E' foo: bar 'A' | 'B' bar: foo 'C' | 'D' rrgrhr z D A C A EDr2r>)typer7rr8r9Ar4r;Cr@rBrQrXErZr\zB C A EB)r rrsrtrrrirjrDgenerateexecgetvaluerrrZassertIsNotNone) r%r&r(rwrxrnsrFrGr)r)r*test_mutually_left_recursiveysB       z&TestPegen.test_mutually_left_recursivecCs|d}t|t}t}t||}|di}t|||d}|t td|Wdn1sn0YdS)Nzf start: target '=' target: maybe '+' | NAME maybe: maybe '-' | target r{r zx - + =) r rrsrtrrrrrarb)r%r&r(rwrxrrFr)r)r*"test_nasty_mutually_left_recursives     z,TestPegen.test_nasty_mutually_left_recursivecCsd}t|}td|}||ttdddddgttddd ddttd d d ddgttd ddddttd ddddggggggggdS)Nz start: (expr_stmt | assign_stmt) &'.' expr_stmt: !(target '=') expr assign_stmt: target '=' expr expr: term ('+' term)* target: NAME term: NUMBER zfoo = 12 + 12 .rgr2r;r6=r@rBZ12rQrZrLr\r^)r3 )r3 )r r rrrr rrJr)r)r*test_lookaheads@  zTestPegen.test_lookaheadcCs:d}|tt|Wdn1s,0YdS)Nz& start: foo=!'x' NAME )rarbr r%r(r)r)r*test_named_lookahead_errors z$TestPegen.test_named_lookahead_errorcCsd}t|dS)NzA start: attr | NAME attr: start '.' NAME r rr)r)r*test_start_leaderszTestPegen.test_start_leadercCsd}t|dS)Nz start: [NAME*] rrr)r)r*test_opt_sequenceszTestPegen.test_opt_sequencecCsPd}|t.}t||dt|jjvWdn1sB0YdS)Nz start: foo foo: bar '+' | baz '+' | '+' bar: baz '-' | foo '-' | '-' baz: foo '*' | bar '*' | '*' z no leader)ra ValueErrorr rDr exceptionvalue)r%r(Zerrinfor)r)r*test_left_recursion_too_complexs z)TestPegen.test_left_recursion_too_complexc CsXd}t|}td|}||ttdddddttdddddgttd dd ddgdS) Nz< start: '(' ~ expr ')' expr: NUMBER z(1)(r2r>r6r=r4)r;)r r rrr rrJr)r)r*test_cuts zTestPegen.test_cutcCs:d}|tt|}Wdn1s,0YdS)Nz< start: foo ENDMARKER foo: bar NAME rarr r%r(rFr)r)r*test_dangling_references z!TestPegen.test_dangling_referencecCs:d}|tt|}Wdn1s,0YdS)Nz/ start: foo foo: NAMEE rrr)r)r*test_bad_token_reference&s z"TestPegen.test_bad_token_referencecCs:d}|tt|}Wdn1s,0YdS)Nz foo: NAME rrr)r)r*test_missing_start.s zTestPegen.test_missing_startcCs<d}|tdt|}Wdn1s.0YdS)Nz< start: _a b _a: 'a' b: 'b' z"cannot start with underscore: '_a'ZassertRaisesRegexrr rr)r)r*test_invalid_rule_name5sz TestPegen.test_invalid_rule_namecCs<d}|tdt|}Wdn1s.0YdS)Nz= start: a b a: _x='a' b: 'b' "cannot start with underscore: '_x'rrr)r)r*test_invalid_variable_name>sz$TestPegen.test_invalid_variable_namecCs<d}|tdt|}Wdn1s.0YdS)NzK start: a b a: (_x='a' | 'b') | 'c' b: 'b' rrrr)r)r*+test_invalid_variable_name_in_temporal_ruleGsz5TestPegen.test_invalid_variable_name_in_temporal_rule)"__name__ __module__ __qualname__r+r,r-rHrKrOrPrSrVr`rcrdrerkrqryrzrrrrrrrrrrrrrrr)r)r)r*rs>   ,1    rc@sbeZdZGdddeZddddZddddZddd d Zddd d Zddd dZ dS)TestGrammarVisitorcs6eZdZddddZeeeddfdd ZZS)zTestGrammarVisitor.VisitorNrcCs d|_dS)Nr)n_nodes)r%r)r)r*__init__Ssz#TestGrammarVisitor.Visitor.__init__)rGargskwargsrcs,|jd7_tj|g|Ri|dS)Nr3)rsupervisit)r%rGrr __class__r)r*rVsz TestGrammarVisitor.Visitor.visit)rrrrrr __classcell__r)r)rr*VisitorRsrNrcCs2d}t|t}|}||||jddS)Nz start: 'a' rRr rrrrrr%r(rZvisitorr)r)r*test_parse_trivial_grammarZs   z-TestGrammarVisitor.test_parse_trivial_grammarcCs2d}t|t}|}||||jddS)Nz5 start: rule rule: 'a' | 'b' rrr)r)r*test_parse_or_grammares   z(TestGrammarVisitor.test_parse_or_grammarcCs2d}t|t}|}||||jddS)Nz start: 'a'+ rYrrr)r)r*test_parse_repeat1_grammarvs   z-TestGrammarVisitor.test_parse_repeat1_grammarcCs2d}t|t}|}||||jddS)Nz start: 'a'* rYrrr)r)r*test_parse_repeat0_grammars   z-TestGrammarVisitor.test_parse_repeat0_grammarcCs2d}t|t}|}||||jddS)Nz" start: 'a' ['b'] rrr)r)r*test_parse_optional_grammars   z.TestGrammarVisitor.test_parse_optional_grammar) rrrrrrrrrrr)r)r)r*rQs    rc@s6eZdZddddZddddZddddZdS) TestGrammarVisualizerNrcCsLd}t|t}t}g}|j||jdd|}td}|||dS)Nz start: 'a' 'b' printerr:u └──Rule └──Rhs └──Alt ├──NamedItem │ └──StringLeaf("'a'") └──NamedItem └──StringLeaf("'b'") r rrZprint_grammar_astappendjoinr!r"rr%r(rrlinesoutputZexpected_outputr)r)r*test_simple_rules   z&TestGrammarVisualizer.test_simple_rulecCsLd}t|t}t}g}|j||jdd|}td}|||dS)Nz: start: a b a: 'a' b: 'b' rr:u; └──Rule └──Rhs └──Alt ├──NamedItem │ └──NameLeaf('a') └──NamedItem └──NameLeaf('b') └──Rule └──Rhs └──Alt └──NamedItem └──StringLeaf("'a'") └──Rule └──Rhs └──Alt └──NamedItem └──StringLeaf("'b'") rrr)r)r*test_multiple_ruless  z)TestGrammarVisualizer.test_multiple_rulescCsLd}t|t}t}g}|j||jdd|}td}|||dS)Nz, start: 'a' ['b'['c'['d']]] rr:u └──Rule └──Rhs └──Alt ├──NamedItem │ └──StringLeaf("'a'") └──NamedItem └──Opt └──Rhs └──Alt ├──NamedItem │ └──StringLeaf("'b'") └──NamedItem └──Opt └──Rhs └──Alt ├──NamedItem │ └──StringLeaf("'c'") └──NamedItem └──Opt └──Rhs └──Alt └──NamedItem └──StringLeaf("'d'") rrr)r)r*test_deep_nested_rules  z+TestGrammarVisualizer.test_deep_nested_rule)rrrrrrr)r)r)r*rs'r)%rsr!Zunittesttestrtypingrrtokenizerrrrr Zskip_if_missingZimports_under_toolZpegen.grammar_parserr rZpegen.testutilr r r Z pegen.grammarrrrZpegen.grammar_visualizerrZ pegen.parserrZpegen.python_generatorrZTestCaserrrr)r)r)r*s(      *>M