parser.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932
  1. # Copyright (c) 2015-2024 Rocky Bernstein
  2. # Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
  3. # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
  4. # Copyright (c) 1999 John Aycock
  5. #
  6. # This program is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation, either version 3 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. """
  19. Common uncompyle6 parser routines.
  20. """
  21. import sys
  22. from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG, GenericASTBuilder
  23. from xdis import iscode
  24. from uncompyle6.show import maybe_show_asm
  25. class ParserError(Exception):
  26. def __init__(self, token, offset, debug=PARSER_DEFAULT_DEBUG):
  27. self.token = token
  28. self.offset = offset
  29. self.debug = debug
  30. def __str__(self):
  31. return "Parse error at or near `%r' instruction at offset %s\n" % (
  32. self.token,
  33. self.offset,
  34. )
  35. def nop_func(self, args):
  36. return None
  37. class PythonParser(GenericASTBuilder):
  38. def __init__(self, syntax_tree_class, start, debug):
  39. super(PythonParser, self).__init__(syntax_tree_class, start, debug)
  40. # FIXME: customize per python parser version
  41. # These are the non-terminals we should collect into a list.
  42. # For example instead of:
  43. # stmts -> stmts stmt -> stmts stmt stmt ...
  44. # collect as stmts -> stmt stmt ...
  45. nt_list = [
  46. "_come_froms",
  47. "_stmts",
  48. "attributes",
  49. "add_consts",
  50. "come_froms",
  51. "except_stmts",
  52. "exprlist",
  53. "importlist",
  54. "key_value_pairs",
  55. "kvlist",
  56. "kwargs",
  57. "l_stmts",
  58. "stmts",
  59. # Python < 3
  60. "print_items",
  61. # PyPy:
  62. "imports_cont",
  63. "kvlist_n",
  64. # Python 3.6+
  65. "come_from_loops",
  66. # Python 3.7+
  67. "importlist37",
  68. # Python < 1.4
  69. "args_store",
  70. ]
  71. self.collect = frozenset(nt_list)
  72. # For these items we need to keep the 1st epslion reduction since
  73. # the nonterminal name is used in a semantic action.
  74. self.keep_epsilon = frozenset(("kvlist_n", "kvlist"))
  75. # ??? Do we need a debug option to skip eliding singleton reductions?
  76. # Time will tell if it if useful in debugging
  77. # FIXME: optional_nt is a misnomer. It's really about there being a
  78. # singleton reduction that we can simplify. It also happens to be optional
  79. # in its other derivation
  80. self.optional_nt |= frozenset(
  81. (
  82. "come_froms",
  83. "suite_stmts",
  84. "l_stmts_opt",
  85. "c_stmts_opt",
  86. "stmts_opt",
  87. "stmt",
  88. )
  89. )
  90. # Reduce singleton reductions in these nonterminals:
  91. # FIXME: would love to do expr, sstmts, stmts and
  92. # so on but that would require major changes to the
  93. # semantic actions
  94. self.singleton = frozenset(
  95. ("str", "store", "_stmts", "suite_stmts_opt", "inplace_op", "add_value")
  96. )
  97. # Instructions filled in from scanner
  98. self.insts = []
  99. self.version = tuple()
  100. def ast_first_offset(self, ast):
  101. if hasattr(ast, "offset"):
  102. return ast.offset
  103. else:
  104. return self.ast_first_offset(ast[0])
  105. def add_unique_rule(self, rule, opname, arg_count, customize):
  106. """Add rule to grammar, but only if it hasn't been added previously
  107. opname and stack_count are used in the customize() semantic
  108. the actions to add the semantic action rule. Stack_count is
  109. used in custom opcodes like MAKE_FUNCTION to indicate how
  110. many arguments it has. Often it is not used.
  111. """
  112. if rule not in self.new_rules:
  113. # print("XXX ", rule) # debug
  114. self.new_rules.add(rule)
  115. self.addRule(rule, nop_func)
  116. customize[opname] = arg_count
  117. pass
  118. return
  119. def add_unique_rules(self, rules, customize):
  120. """Add rules (a list of string) to grammar. Note that
  121. the rules must not be those that set arg_count in the
  122. custom dictionary.
  123. """
  124. for rule in rules:
  125. if len(rule) == 0:
  126. continue
  127. opname = rule.split("::=")[0].strip()
  128. self.add_unique_rule(rule, opname, 0, customize)
  129. return
  130. def add_unique_doc_rules(self, rules_str, customize):
  131. """Add rules (a docstring-like list of rules) to grammar.
  132. Note that the rules must not be those that set arg_count in the
  133. custom dictionary.
  134. """
  135. rules = [r.strip() for r in rules_str.split("\n")]
  136. self.add_unique_rules(rules, customize)
  137. return
  138. def cleanup(self):
  139. """
  140. Remove recursive references to allow garbage
  141. collector to collect this object.
  142. """
  143. for rule_dict in (self.rule2func, self.rules, self.rule2name):
  144. for i in list(rule_dict.keys()):
  145. rule_dict[i] = None
  146. for i in dir(self):
  147. setattr(self, i, None)
  148. def debug_reduce(self, rule, tokens, parent, last_token_pos):
  149. """Customized format and print for our kind of tokens
  150. which gets called in debugging grammar reduce rules
  151. """
  152. def fix(c):
  153. s = str(c)
  154. token_pos = s.find("_")
  155. if token_pos == -1:
  156. return s
  157. else:
  158. return s[:token_pos]
  159. prefix = ""
  160. if parent and tokens:
  161. p_token = tokens[parent]
  162. if hasattr(p_token, "linestart") and p_token.linestart:
  163. prefix = "L.%3d: " % p_token.linestart
  164. else:
  165. prefix = " "
  166. if hasattr(p_token, "offset"):
  167. prefix += "%3s" % fix(p_token.offset)
  168. if len(rule[1]) > 1:
  169. prefix += "-%-3s " % fix(tokens[last_token_pos - 1].offset)
  170. else:
  171. prefix += " "
  172. else:
  173. prefix = " "
  174. print("%s%s ::= %s (%d)" % (prefix, rule[0], " ".join(rule[1]), last_token_pos))
  175. def error(self, instructions, index):
  176. # Find the last line boundary
  177. start, finish = -1, -1
  178. for start in range(index, -1, -1):
  179. if instructions[start].linestart:
  180. break
  181. pass
  182. for finish in range(index + 1, len(instructions)):
  183. if instructions[finish].linestart:
  184. break
  185. pass
  186. if start >= 0:
  187. err_token = instructions[index]
  188. print("Instruction context:")
  189. for i in range(start, finish):
  190. if i != index:
  191. indent = " "
  192. else:
  193. indent = "-> "
  194. print("%s%s" % (indent, instructions[i]))
  195. raise ParserError(err_token, err_token.offset, self.debug["reduce"])
  196. else:
  197. raise ParserError(None, -1, self.debug["reduce"])
  198. def get_pos_kw(self, token):
  199. """
  200. Return then the number of positional parameters and keyword
  201. parfameters represented by the attr (operand) field of
  202. token.
  203. This appears in CALL_FUNCTION or CALL_METHOD (PyPy) tokens
  204. """
  205. # Low byte indicates number of positional parameters,
  206. # high byte number of keyword parameters
  207. assert token.kind.startswith("CALL_FUNCTION") or token.kind.startswith(
  208. "CALL_METHOD"
  209. )
  210. args_pos = token.attr & 0xFF
  211. args_kw = (token.attr >> 8) & 0xFF
  212. return args_pos, args_kw
  213. def nonterminal(self, nt, args):
  214. n = len(args)
  215. # # Use this to find lots of singleton rule
  216. # if n == 1 and nt not in self.singleton:
  217. # print("XXX", nt)
  218. if nt in self.collect and n > 1:
  219. #
  220. # Collect iterated thingies together. That is rather than
  221. # stmts -> stmts stmt -> stmts stmt -> ...
  222. # stmms -> stmt stmt ...
  223. #
  224. if not hasattr(args[0], "append"):
  225. # Was in self.optional_nt as a single item, but we find we have
  226. # more than one now...
  227. rv = GenericASTBuilder.nonterminal(self, nt, [args[0]])
  228. else:
  229. rv = args[0]
  230. pass
  231. # In a list-like entity where the first item goes to epsilon,
  232. # drop that and save the 2nd item as the first one
  233. if len(rv) == 0 and nt not in self.keep_epsilon:
  234. rv = args[1]
  235. else:
  236. rv.append(args[1])
  237. elif n == 1 and args[0] in self.singleton:
  238. rv = GenericASTBuilder.nonterminal(self, nt, args[0])
  239. del args[0] # save memory
  240. elif n == 1 and nt in self.optional_nt:
  241. rv = args[0]
  242. else:
  243. rv = GenericASTBuilder.nonterminal(self, nt, args)
  244. return rv
  245. def __ambiguity(self, children):
  246. # only for debugging! to be removed hG/2000-10-15
  247. print(children)
  248. return GenericASTBuilder.ambiguity(self, children)
  249. def resolve(self, rule: list):
  250. if len(rule) == 2 and "function_def" in rule and "assign" in rule:
  251. return "function_def"
  252. if "grammar" in rule and "expr" in rule:
  253. return "expr"
  254. # print >> sys.stderr, 'resolve', str(rule)
  255. return GenericASTBuilder.resolve(self, rule)
  256. ###############################################
  257. # Common Python 2 and Python 3 grammar rules #
  258. ###############################################
  259. def p_start(self, args):
  260. """
  261. # The start or goal symbol
  262. stmts ::= sstmt+
  263. """
  264. def p_call_stmt(self, args):
  265. """
  266. # eval-mode compilation. Single-mode interactive compilation
  267. # adds another rule.
  268. call_stmt ::= expr POP_TOP
  269. """
  270. def p_stmt(self, args):
  271. """
  272. pass ::=
  273. _stmts ::= stmt+
  274. # statements with continue
  275. c_stmts ::= _stmts
  276. c_stmts ::= _stmts lastc_stmt
  277. c_stmts ::= lastc_stmt
  278. c_stmts ::= continues
  279. ending_return ::= RETURN_VALUE RETURN_LAST
  280. ending_return ::= RETURN_VALUE_LAMBDA LAMBDA_MARKER
  281. lastc_stmt ::= iflaststmt
  282. lastc_stmt ::= forelselaststmt
  283. lastc_stmt ::= ifelsestmtc
  284. c_stmts_opt ::= c_stmts
  285. c_stmts_opt ::= pass
  286. stmts_opt ::= _stmts
  287. stmts_opt ::= pass
  288. # statements inside a loop
  289. l_stmts ::= _stmts
  290. l_stmts ::= returns
  291. l_stmts ::= continues
  292. l_stmts ::= _stmts lastl_stmt
  293. l_stmts ::= lastl_stmt
  294. lastl_stmt ::= iflaststmtl
  295. lastl_stmt ::= ifelsestmtl
  296. lastl_stmt ::= forelselaststmtl
  297. lastl_stmt ::= tryelsestmtl
  298. l_stmts_opt ::= l_stmts
  299. l_stmts_opt ::= pass
  300. suite_stmts ::= _stmts
  301. suite_stmts ::= returns
  302. suite_stmts ::= continues
  303. suite_stmts_opt ::= suite_stmts
  304. # passtmt is needed for semantic actions to add "pass"
  305. suite_stmts_opt ::= pass
  306. else_suite ::= suite_stmts
  307. else_suitel ::= l_stmts
  308. else_suitec ::= c_stmts
  309. else_suitec ::= returns
  310. stmt ::= assert
  311. stmt ::= classdef
  312. stmt ::= call_stmt
  313. stmt ::= ifstmt
  314. stmt ::= ifelsestmt
  315. stmt ::= whilestmt
  316. stmt ::= while1stmt
  317. stmt ::= whileelsestmt
  318. stmt ::= while1elsestmt
  319. stmt ::= for
  320. stmt ::= forelsestmt
  321. stmt ::= try_except
  322. stmt ::= tryelsestmt
  323. stmt ::= tryfinallystmt
  324. stmt ::= with
  325. stmt ::= with_as
  326. stmt ::= delete
  327. delete ::= DELETE_FAST
  328. delete ::= DELETE_NAME
  329. delete ::= DELETE_GLOBAL
  330. stmt ::= return
  331. return ::= return_expr RETURN_VALUE
  332. # "returns" nonterminal is a sequence of statements that ends in a RETURN statement.
  333. # In later Python versions with jump optimization, this can cause JUMPs
  334. # that would normally appear to be omitted.
  335. returns ::= return
  336. returns ::= _stmts return
  337. # NOP
  338. stmt ::= nop_stmt
  339. nop_stmt ::= NOP
  340. """
  341. pass
  342. def p_function_def(self, args):
  343. """
  344. stmt ::= function_def
  345. function_def ::= mkfunc store
  346. stmt ::= function_def_deco
  347. function_def_deco ::= mkfuncdeco store
  348. mkfuncdeco ::= expr mkfuncdeco CALL_FUNCTION_1
  349. mkfuncdeco ::= expr mkfuncdeco0 CALL_FUNCTION_1
  350. mkfuncdeco0 ::= mkfunc
  351. load_closure ::= load_closure LOAD_CLOSURE
  352. load_closure ::= LOAD_CLOSURE
  353. """
  354. def p_generator_exp(self, args):
  355. """
  356. expr ::= generator_exp
  357. stmt ::= genexpr_func
  358. genexpr_func ::= LOAD_FAST FOR_ITER store comp_iter JUMP_BACK
  359. """
  360. def p_jump(self, args):
  361. """
  362. _jump ::= JUMP_ABSOLUTE
  363. _jump ::= JUMP_FORWARD
  364. _jump ::= JUMP_BACK
  365. # Zero or more COME_FROMs - loops can have this
  366. _come_froms ::= COME_FROM*
  367. # One or more COME_FROMs - joins of tryelse's have this
  368. come_froms ::= COME_FROM+
  369. # Zero or one COME_FROM
  370. # And/or expressions have this
  371. come_from_opt ::= COME_FROM?
  372. """
  373. def p_augmented_assign(self, args):
  374. """
  375. stmt ::= aug_assign1
  376. stmt ::= aug_assign2
  377. # This is odd in that other aug_assign1's have only 3 slots
  378. # The store isn't used as that's supposed to be also
  379. # indicated in the first expr
  380. aug_assign1 ::= expr expr
  381. inplace_op store
  382. aug_assign1 ::= expr expr
  383. inplace_op ROT_THREE STORE_SUBSCR
  384. aug_assign2 ::= expr DUP_TOP LOAD_ATTR expr
  385. inplace_op ROT_TWO STORE_ATTR
  386. inplace_op ::= INPLACE_ADD
  387. inplace_op ::= INPLACE_SUBTRACT
  388. inplace_op ::= INPLACE_MULTIPLY
  389. inplace_op ::= INPLACE_TRUE_DIVIDE
  390. inplace_op ::= INPLACE_FLOOR_DIVIDE
  391. inplace_op ::= INPLACE_MODULO
  392. inplace_op ::= INPLACE_POWER
  393. inplace_op ::= INPLACE_LSHIFT
  394. inplace_op ::= INPLACE_RSHIFT
  395. inplace_op ::= INPLACE_AND
  396. inplace_op ::= INPLACE_XOR
  397. inplace_op ::= INPLACE_OR
  398. """
  399. def p_assign(self, args):
  400. """
  401. stmt ::= assign
  402. assign ::= expr DUP_TOP designList
  403. assign ::= expr store
  404. stmt ::= assign2
  405. stmt ::= assign3
  406. assign2 ::= expr expr ROT_TWO store store
  407. assign3 ::= expr expr expr ROT_THREE ROT_TWO store store store
  408. """
  409. def p_forstmt(self, args):
  410. """
  411. for_iter ::= GET_ITER FOR_ITER
  412. for_block ::= l_stmts_opt _come_froms JUMP_BACK
  413. forelsestmt ::= SETUP_LOOP expr for_iter store
  414. for_block POP_BLOCK else_suite _come_froms
  415. forelselaststmt ::= SETUP_LOOP expr for_iter store
  416. for_block POP_BLOCK else_suitec _come_froms
  417. forelselaststmtl ::= SETUP_LOOP expr for_iter store
  418. for_block POP_BLOCK else_suitel _come_froms
  419. """
  420. def p_import20(self, args):
  421. """
  422. stmt ::= import
  423. stmt ::= import_from
  424. stmt ::= import_from_star
  425. stmt ::= importmultiple
  426. importlist ::= importlist alias
  427. importlist ::= alias
  428. alias ::= IMPORT_NAME store
  429. alias ::= IMPORT_FROM store
  430. alias ::= IMPORT_NAME attributes store
  431. import ::= LOAD_CONST LOAD_CONST alias
  432. import_from_star ::= LOAD_CONST LOAD_CONST IMPORT_NAME IMPORT_STAR
  433. import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist POP_TOP
  434. importmultiple ::= LOAD_CONST LOAD_CONST alias imports_cont
  435. imports_cont ::= import_cont+
  436. import_cont ::= LOAD_CONST LOAD_CONST alias
  437. attributes ::= LOAD_ATTR+
  438. """
  439. def p_list_comprehension(self, args):
  440. """
  441. expr ::= list_comp
  442. list_iter ::= list_for
  443. list_iter ::= list_if
  444. list_iter ::= list_if_not
  445. list_iter ::= lc_body
  446. list_if ::= expr jmp_false list_iter
  447. list_if_not ::= expr jmp_true list_iter
  448. """
  449. def p_set_comp(self, args):
  450. """
  451. comp_iter ::= comp_for
  452. comp_iter ::= comp_body
  453. comp_body ::= gen_comp_body
  454. gen_comp_body ::= expr YIELD_VALUE POP_TOP
  455. comp_iter ::= comp_if
  456. comp_if ::= expr jmp_false comp_iter
  457. """
  458. def p_expr(self, args):
  459. """
  460. expr ::= LOAD_CODE
  461. expr ::= LOAD_CONST
  462. expr ::= LOAD_DEREF
  463. expr ::= LOAD_FAST
  464. expr ::= LOAD_GLOBAL
  465. expr ::= LOAD_NAME
  466. expr ::= _lambda_body
  467. expr ::= and
  468. expr ::= bin_op
  469. expr ::= call
  470. expr ::= compare
  471. expr ::= dict
  472. expr ::= list
  473. expr ::= or
  474. expr ::= subscript
  475. expr ::= subscript2
  476. expr ::= unary_op
  477. expr ::= unary_not
  478. expr ::= yield
  479. # bin_op (formerly "binary_expr") is the Python AST BinOp
  480. bin_op ::= expr expr binary_operator
  481. binary_operator ::= BINARY_ADD
  482. binary_operator ::= BINARY_MULTIPLY
  483. binary_operator ::= BINARY_AND
  484. binary_operator ::= BINARY_OR
  485. binary_operator ::= BINARY_XOR
  486. binary_operator ::= BINARY_SUBTRACT
  487. binary_operator ::= BINARY_TRUE_DIVIDE
  488. binary_operator ::= BINARY_FLOOR_DIVIDE
  489. binary_operator ::= BINARY_MODULO
  490. binary_operator ::= BINARY_LSHIFT
  491. binary_operator ::= BINARY_RSHIFT
  492. binary_operator ::= BINARY_POWER
  493. # unary_op (formerly "unary_expr") is the Python AST BinOp
  494. unary_op ::= expr unary_operator
  495. unary_operator ::= UNARY_POSITIVE
  496. unary_operator ::= UNARY_NEGATIVE
  497. unary_operator ::= UNARY_INVERT
  498. unary_not ::= expr UNARY_NOT
  499. subscript ::= expr expr BINARY_SUBSCR
  500. attribute ::= expr LOAD_ATTR
  501. get_iter ::= expr GET_ITER
  502. yield ::= expr YIELD_VALUE
  503. _lambda_body ::= lambda_body
  504. expr ::= if_exp
  505. return_expr ::= expr
  506. return_expr ::= ret_and
  507. return_expr ::= ret_or
  508. return_expr_or_cond ::= return_expr
  509. return_expr_or_cond ::= if_exp_ret
  510. stmt ::= return_expr_lambda
  511. return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
  512. return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA
  513. compare ::= compare_chained
  514. compare ::= compare_single
  515. compare_single ::= expr expr COMPARE_OP
  516. # A compare_chained is two comparisons, as in: x <= y <= z
  517. compare_chained ::= expr compared_chained_middle ROT_TWO POP_TOP
  518. _come_froms
  519. compare_chained_right ::= expr COMPARE_OP JUMP_FORWARD
  520. # Non-null kvlist items are broken out in the individual grammars
  521. kvlist ::=
  522. # Positional arguments in make_function
  523. pos_arg ::= expr
  524. """
  525. def p_store(self, args):
  526. """
  527. # Note. The below is right-recursive:
  528. designList ::= store store
  529. designList ::= store DUP_TOP designList
  530. ## Can we replace with left-recursive, and redo with:
  531. ##
  532. ## designList ::= designLists store store
  533. ## designLists ::= designLists store DUP_TOP
  534. ## designLists ::=
  535. ## Will need to redo semantic actiion
  536. store ::= STORE_FAST
  537. store ::= STORE_NAME
  538. store ::= STORE_GLOBAL
  539. store ::= STORE_DEREF
  540. store ::= expr STORE_ATTR
  541. store ::= store_subscript
  542. store_subscript ::= expr expr STORE_SUBSCR
  543. store ::= unpack
  544. """
  545. def parse(p, tokens, customize, code):
  546. p.customize_grammar_rules(tokens, customize)
  547. ast = p.parse(tokens)
  548. # p.cleanup()
  549. return ast
  550. def get_python_parser(
  551. version, debug_parser=PARSER_DEFAULT_DEBUG, compile_mode="exec", is_pypy=False
  552. ):
  553. """Returns parser object for Python version 2 or 3, 3.2, 3.5on,
  554. etc., depending on the parameters passed. *compile_mode* is either
  555. 'exec', 'eval', or 'single'. See
  556. https://docs.python.org/3.6/library/functions.html#compile for an
  557. explanation of the different modes.
  558. """
  559. # If version is a string, turn that into the corresponding float.
  560. if isinstance(version, str):
  561. version = tuple([int(v) for v in version.split(".")[:2]])
  562. version = version[:2]
  563. p = None
  564. # FIXME: there has to be a better way...
  565. # We could do this as a table lookup, but that would force us
  566. # in import all of the parsers all of the time. Perhaps there is
  567. # a lazy way of doing the import?
  568. if version < (3, 0):
  569. if version < (2, 2):
  570. if version == (1, 0):
  571. import uncompyle6.parsers.parse10 as parse10
  572. if compile_mode == "exec":
  573. p = parse10.Python10Parser(debug_parser)
  574. else:
  575. p = parse10.Python10ParserSingle(debug_parser)
  576. elif version == (1, 1):
  577. import uncompyle6.parsers.parse11 as parse11
  578. if compile_mode == "exec":
  579. p = parse11.Python11Parser(debug_parser)
  580. else:
  581. p = parse11.Python11ParserSingle(debug_parser)
  582. if version == (1, 2):
  583. import uncompyle6.parsers.parse12 as parse12
  584. if compile_mode == "exec":
  585. p = parse12.Python12Parser(debug_parser)
  586. else:
  587. p = parse12.Python12ParserSingle(debug_parser)
  588. if version == (1, 3):
  589. import uncompyle6.parsers.parse13 as parse13
  590. if compile_mode == "exec":
  591. p = parse13.Python13Parser(debug_parser)
  592. else:
  593. p = parse13.Python13ParserSingle(debug_parser)
  594. elif version == (1, 4):
  595. import uncompyle6.parsers.parse14 as parse14
  596. if compile_mode == "exec":
  597. p = parse14.Python14Parser(debug_parser)
  598. else:
  599. p = parse14.Python14ParserSingle(debug_parser)
  600. elif version == (1, 5):
  601. import uncompyle6.parsers.parse15 as parse15
  602. if compile_mode == "exec":
  603. p = parse15.Python15Parser(debug_parser)
  604. else:
  605. p = parse15.Python15ParserSingle(debug_parser)
  606. elif version == (1, 6):
  607. import uncompyle6.parsers.parse16 as parse16
  608. if compile_mode == "exec":
  609. p = parse16.Python16Parser(debug_parser)
  610. else:
  611. p = parse16.Python16ParserSingle(debug_parser)
  612. elif version == (2, 1):
  613. import uncompyle6.parsers.parse21 as parse21
  614. if compile_mode == "exec":
  615. p = parse21.Python21Parser(debug_parser)
  616. else:
  617. p = parse21.Python21ParserSingle(debug_parser)
  618. elif version == (2, 2):
  619. import uncompyle6.parsers.parse22 as parse22
  620. if compile_mode == "exec":
  621. p = parse22.Python22Parser(debug_parser)
  622. else:
  623. p = parse22.Python22ParserSingle(debug_parser)
  624. elif version == (2, 3):
  625. import uncompyle6.parsers.parse23 as parse23
  626. if compile_mode == "exec":
  627. p = parse23.Python23Parser(debug_parser)
  628. else:
  629. p = parse23.Python23ParserSingle(debug_parser)
  630. elif version == (2, 4):
  631. import uncompyle6.parsers.parse24 as parse24
  632. if compile_mode == "exec":
  633. p = parse24.Python24Parser(debug_parser)
  634. else:
  635. p = parse24.Python24ParserSingle(debug_parser)
  636. elif version == (2, 5):
  637. import uncompyle6.parsers.parse25 as parse25
  638. if compile_mode == "exec":
  639. p = parse25.Python25Parser(debug_parser)
  640. else:
  641. p = parse25.Python25ParserSingle(debug_parser)
  642. elif version == (2, 6):
  643. import uncompyle6.parsers.parse26 as parse26
  644. if compile_mode == "exec":
  645. p = parse26.Python26Parser(debug_parser)
  646. else:
  647. p = parse26.Python26ParserSingle(debug_parser)
  648. elif version == (2, 7):
  649. import uncompyle6.parsers.parse27 as parse27
  650. if compile_mode == "exec":
  651. p = parse27.Python27Parser(debug_parser)
  652. else:
  653. p = parse27.Python27ParserSingle(debug_parser)
  654. else:
  655. import uncompyle6.parsers.parse2 as parse2
  656. if compile_mode == "exec":
  657. p = parse2.Python2Parser(debug_parser)
  658. else:
  659. p = parse2.Python2ParserSingle(debug_parser)
  660. pass
  661. pass
  662. pass
  663. else:
  664. import uncompyle6.parsers.parse3 as parse3
  665. if version == (3, 0):
  666. import uncompyle6.parsers.parse30 as parse30
  667. if compile_mode == "exec":
  668. p = parse30.Python30Parser(debug_parser)
  669. else:
  670. p = parse30.Python30ParserSingle(debug_parser)
  671. elif version == (3, 1):
  672. import uncompyle6.parsers.parse31 as parse31
  673. if compile_mode == "exec":
  674. p = parse31.Python31Parser(debug_parser)
  675. else:
  676. p = parse31.Python31ParserSingle(debug_parser)
  677. elif version == (3, 2):
  678. import uncompyle6.parsers.parse32 as parse32
  679. if compile_mode == "exec":
  680. p = parse32.Python32Parser(debug_parser)
  681. else:
  682. p = parse32.Python32ParserSingle(debug_parser)
  683. elif version == (3, 3):
  684. import uncompyle6.parsers.parse33 as parse33
  685. if compile_mode == "exec":
  686. p = parse33.Python33Parser(debug_parser)
  687. else:
  688. p = parse33.Python33ParserSingle(debug_parser)
  689. elif version == (3, 4):
  690. import uncompyle6.parsers.parse34 as parse34
  691. if compile_mode == "exec":
  692. p = parse34.Python34Parser(debug_parser)
  693. else:
  694. p = parse34.Python34ParserSingle(debug_parser)
  695. elif version == (3, 5):
  696. import uncompyle6.parsers.parse35 as parse35
  697. if compile_mode == "exec":
  698. p = parse35.Python35Parser(debug_parser)
  699. else:
  700. p = parse35.Python35ParserSingle(debug_parser)
  701. elif version == (3, 6):
  702. import uncompyle6.parsers.parse36 as parse36
  703. if compile_mode == "exec":
  704. p = parse36.Python36Parser(debug_parser)
  705. else:
  706. p = parse36.Python36ParserSingle(debug_parser)
  707. elif version == (3, 7):
  708. import uncompyle6.parsers.parse37 as parse37
  709. if compile_mode == "exec":
  710. p = parse37.Python37Parser(debug_parser)
  711. else:
  712. p = parse37.Python37ParserSingle(debug_parser)
  713. elif version == (3, 8):
  714. import uncompyle6.parsers.parse38 as parse38
  715. if compile_mode == "exec":
  716. p = parse38.Python38Parser(debug_parser)
  717. else:
  718. p = parse38.Python38ParserSingle(debug_parser)
  719. else:
  720. if compile_mode == "exec":
  721. p = parse3.Python3Parser(debug_parser)
  722. else:
  723. p = parse3.Python3ParserSingle(debug_parser)
  724. p.version = version
  725. # p.dump_grammar() # debug
  726. return p
  727. class PythonParserSingle(PythonParser):
  728. def p_call_stmt_single(self, args):
  729. """
  730. # single-mode compilation. Eval-mode interactive compilation
  731. # drops the last rule.
  732. call_stmt ::= expr PRINT_EXPR
  733. """
  734. def python_parser(
  735. version,
  736. co,
  737. out=sys.stdout,
  738. showasm=False,
  739. parser_debug=PARSER_DEFAULT_DEBUG,
  740. is_pypy=False,
  741. ):
  742. """
  743. Parse a code object to an abstract syntax tree representation.
  744. :param version: The python version this code is from as a float, for
  745. example 2.6, 2.7, 3.2, 3.3, 3.4, 3.5 etc.
  746. :param co: The code object to parse.
  747. :param out: File like object to write the output to.
  748. :param showasm: Flag which determines whether the disassembled and
  749. ingested code is written to sys.stdout or not.
  750. :param parser_debug: dict containing debug flags for the spark parser.
  751. :param is_pypy: True if we are running PyPY
  752. :return: Abstract syntax tree representation of the code object.
  753. """
  754. assert iscode(co)
  755. from uncompyle6.scanner import get_scanner
  756. scanner = get_scanner(version, is_pypy)
  757. tokens, customize = scanner.ingest(co)
  758. maybe_show_asm(showasm, tokens)
  759. # For heavy grammar debugging
  760. # parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
  761. # 'showstack': 'full'}
  762. p = get_python_parser(version, parser_debug)
  763. # FIXME: have p.insts update in a better way
  764. # modularity is broken here
  765. p.insts = scanner.insts
  766. p.offset2inst_index = scanner.offset2inst_index
  767. return parse(p, tokens, customize, co)
  768. if __name__ == "__main__":
  769. def parse_test(co):
  770. from xdis import IS_PYPY, PYTHON_VERSION_TRIPLE
  771. ast = python_parser(PYTHON_VERSION_TRIPLE, co, showasm=True, is_pypy=IS_PYPY)
  772. print(ast)
  773. return
  774. parse_test(parse_test.__code__)