lambda_custom.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774
  1. # Copyright (c) 2020-2024 Rocky Bernstein
  2. #
  3. # This program is free software: you can redistribute it and/or modify
  4. # it under the terms of the GNU General Public License as published by
  5. # the Free Software Foundation, either version 3 of the License, or
  6. # (at your option) any later version.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License
  14. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  15. """
  16. Grammar Customization rules for Python 3.8's Lambda expression grammar.
  17. """
  18. from decompyle3.parsers.p37.base import Python37BaseParser
  19. from decompyle3.parsers.p38pypy.base import Python38PyPyBaseParser
  20. from decompyle3.parsers.parse_heads import nop_func
  21. class Python38PyPyLambdaCustom(Python38PyPyBaseParser):
  22. def __init__(self):
  23. self.new_rules = set()
  24. self.customized = {}
  25. def remove_rules_pypy38(self):
  26. self.remove_rules(
  27. """
  28. """
  29. )
  30. def customize_grammar_rules_lambda38(self, tokens, customize):
  31. Python38PyPyBaseParser.customize_grammar_rules38(self, tokens, customize)
  32. # self.remove_rules_pypy38()
  33. self.check_reduce["call_kw"] = "AST"
  34. # For a rough break out on the first word. This may
  35. # include instructions that don't need customization,
  36. # but we'll do a finer check after the rough breakout.
  37. customize_instruction_basenames = frozenset(
  38. (
  39. "BEFORE",
  40. "BUILD",
  41. "CALL",
  42. "DICT",
  43. "GET",
  44. "FORMAT",
  45. "LIST",
  46. "LOAD",
  47. "MAKE",
  48. "SETUP",
  49. "UNPACK",
  50. )
  51. )
  52. # Opcode names in the custom_ops_processed set have rules that get added
  53. # unconditionally and the rules are constant. So they need to be done
  54. # only once and if we see the opcode a second we don't have to consider
  55. # adding more rules.
  56. #
  57. custom_ops_processed = frozenset()
  58. # A set of instruction operation names that exist in the token stream.
  59. # We use this customize the grammar that we create.
  60. # 2.6-compatible set comprehensions
  61. self.seen_ops = frozenset([t.kind for t in tokens])
  62. self.seen_op_basenames = frozenset(
  63. [opname[: opname.rfind("_")] for opname in self.seen_ops]
  64. )
  65. custom_ops_processed = {"DICT_MERGE"}
  66. # Loop over instructions adding custom grammar rules based on
  67. # a specific instruction seen.
  68. if "PyPy" in customize:
  69. self.addRule(
  70. """
  71. stmt ::= assign3_pypy
  72. stmt ::= assign2_pypy
  73. assign3_pypy ::= expr expr expr store store store
  74. assign2_pypy ::= expr expr store store
  75. """,
  76. nop_func,
  77. )
  78. n = len(tokens)
  79. # Determine if we have an iteration CALL_FUNCTION_1.
  80. has_get_iter_call_function1 = False
  81. for i, token in enumerate(tokens):
  82. if (
  83. token == "GET_ITER"
  84. and i < n - 2
  85. and self.call_fn_name(tokens[i + 1]) == "CALL_FUNCTION_1"
  86. ):
  87. has_get_iter_call_function1 = True
  88. for i, token in enumerate(tokens):
  89. opname = token.kind
  90. # Do a quick breakout before testing potentially
  91. # each of the dozen or so instruction in if elif.
  92. if (
  93. opname[: opname.find("_")] not in customize_instruction_basenames
  94. or opname in custom_ops_processed
  95. ):
  96. continue
  97. opname_base = opname[: opname.rfind("_")]
  98. # Do a quick breakout before testing potentially
  99. # each of the dozen or so instruction in if elif.
  100. if (
  101. opname[: opname.find("_")] not in customize_instruction_basenames
  102. or opname in custom_ops_processed
  103. ):
  104. continue
  105. if opname == "BEFORE_ASYNC_WITH":
  106. rules_str = """
  107. async_with_post ::= COME_FROM_ASYNC_WITH
  108. WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
  109. WITH_CLEANUP_FINISH END_FINALLY
  110. stmt ::= async_with_as_stmt
  111. async_with_as_stmt ::= expr
  112. async_with_pre
  113. store
  114. suite_stmts_opt
  115. POP_BLOCK LOAD_CONST
  116. async_with_post
  117. async_with_stmt ::= expr
  118. async_with_pre
  119. POP_TOP
  120. c_suite_stmts
  121. POP_BLOCK
  122. BEGIN_FINALLY
  123. WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
  124. WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
  125. POP_BLOCK
  126. BEGIN_FINALLY
  127. COME_FROM_ASYNC_WITH
  128. WITH_CLEANUP_START
  129. GET_AWAITABLE
  130. LOAD_CONST
  131. YIELD_FROM
  132. WITH_CLEANUP_FINISH
  133. END_FINALLY
  134. async_with_stmt ::= expr
  135. async_with_pre
  136. POP_TOP
  137. suite_stmts_opt
  138. POP_BLOCK LOAD_CONST
  139. async_with_post
  140. async_with_stmt ::= expr
  141. async_with_pre
  142. POP_TOP
  143. suite_stmts_opt
  144. async_with_post
  145. """
  146. self.addRule(rules_str, nop_func)
  147. elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
  148. if opname == "BUILD_CONST_DICT":
  149. rule = f"""
  150. add_consts ::= ADD_VALUE*
  151. const_list ::= COLLECTION_START add_consts {opname}
  152. dict ::= const_list
  153. expr ::= dict
  154. """
  155. else:
  156. rule = f"""
  157. add_consts ::= ADD_VALUE*
  158. const_list ::= COLLECTION_START add_consts {opname}
  159. expr ::= const_list
  160. """
  161. self.addRule(rule, nop_func)
  162. elif opname_base in (
  163. "BUILD_LIST",
  164. "BUILD_SET",
  165. "BUILD_SET_UNPACK",
  166. "BUILD_TUPLE",
  167. "BUILD_TUPLE_UNPACK",
  168. ):
  169. v = token.attr
  170. is_LOAD_CLOSURE = False
  171. if opname_base == "BUILD_TUPLE":
  172. # If is part of a "load_closure", then it is not part of a
  173. # "list".
  174. is_LOAD_CLOSURE = True
  175. for j in range(v):
  176. if tokens[i - j - 1].kind != "LOAD_CLOSURE":
  177. is_LOAD_CLOSURE = False
  178. break
  179. if is_LOAD_CLOSURE:
  180. rule = "load_closure ::= %s%s" % (("LOAD_CLOSURE " * v), opname)
  181. self.add_unique_rule(rule, opname, token.attr, customize)
  182. elif opname_base == "BUILD_LIST":
  183. v = token.attr
  184. if v == 0:
  185. rule_str = """
  186. list ::= BUILD_LIST_0
  187. list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND
  188. list ::= list_unpack
  189. """
  190. self.add_unique_doc_rules(rule_str, customize)
  191. elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL":
  192. # FIXME: should this be parameterized by EX value?
  193. self.addRule(
  194. """expr ::= call_ex_kw3
  195. call_ex_kw3 ::= expr
  196. build_tuple_unpack_with_call
  197. expr
  198. CALL_FUNCTION_EX_KW
  199. """,
  200. nop_func,
  201. )
  202. if not is_LOAD_CLOSURE or v == 0:
  203. # We do this complicated test to speed up parsing of
  204. # pathelogically long literals, especially those over 1024.
  205. build_count = token.attr
  206. thousands = build_count // 1024
  207. thirty32s = (build_count // 32) % 32
  208. if thirty32s > 0:
  209. rule = "expr32 ::=%s" % (" expr" * 32)
  210. self.add_unique_rule(rule, opname_base, build_count, customize)
  211. pass
  212. if thousands > 0:
  213. self.add_unique_rule(
  214. "expr1024 ::=%s" % (" expr32" * 32),
  215. opname_base,
  216. build_count,
  217. customize,
  218. )
  219. pass
  220. collection = opname_base[opname_base.find("_") + 1 :].lower()
  221. rule = (
  222. ("%s ::= " % collection)
  223. + "expr1024 " * thousands
  224. + "expr32 " * thirty32s
  225. + "expr " * (build_count % 32)
  226. + opname
  227. )
  228. self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
  229. continue
  230. continue
  231. elif opname.startswith("BUILD_STRING"):
  232. v = token.attr
  233. rules_str = """
  234. expr ::= joined_str
  235. joined_str ::= %sBUILD_STRING_%d
  236. """ % (
  237. "expr " * v,
  238. v,
  239. )
  240. self.add_unique_doc_rules(rules_str, customize)
  241. if "FORMAT_VALUE_ATTR" in self.seen_ops:
  242. rules_str = """
  243. formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
  244. expr ::= formatted_value_attr
  245. """
  246. self.add_unique_doc_rules(rules_str, customize)
  247. elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
  248. v = token.attr
  249. rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
  250. self.addRule(rule, nop_func)
  251. elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
  252. v = token.attr
  253. rule = (
  254. "build_tuple_unpack_with_call ::= "
  255. + "expr1024 " * int(v // 1024)
  256. + "expr32 " * int((v // 32) % 32)
  257. + "expr " * (v % 32)
  258. + opname
  259. )
  260. self.addRule(rule, nop_func)
  261. rule = "starred ::= %s %s" % ("expr " * v, opname)
  262. self.addRule(rule, nop_func)
  263. elif opname == "FORMAT_VALUE":
  264. rules_str = """
  265. expr ::= formatted_value1
  266. formatted_value1 ::= expr FORMAT_VALUE
  267. """
  268. self.add_unique_doc_rules(rules_str, customize)
  269. elif opname == "FORMAT_VALUE_ATTR":
  270. rules_str = """
  271. expr ::= formatted_value2
  272. formatted_value2 ::= expr expr FORMAT_VALUE_ATTR
  273. """
  274. self.add_unique_doc_rules(rules_str, customize)
  275. elif opname == "GET_AITER":
  276. self.add_unique_doc_rules("get_aiter ::= expr GET_AITER", customize)
  277. if not {"MAKE_FUNCTION_0", "MAKE_FUNCTION_CLOSURE"} in self.seen_ops:
  278. self.addRule(
  279. """
  280. expr ::= dict_comp_async
  281. expr ::= generator_exp_async
  282. expr ::= list_comp_async
  283. dict_comp_async ::= LOAD_DICTCOMP
  284. LOAD_STR
  285. MAKE_FUNCTION_0
  286. get_aiter
  287. CALL_FUNCTION_1
  288. dict_comp_async ::= BUILD_MAP_0 LOAD_ARG
  289. dict_comp_async
  290. generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0
  291. get_aiter CALL_FUNCTION_1
  292. list_comp_async ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0
  293. get_aiter CALL_FUNCTION_1
  294. await
  295. list_comp_async ::= LOAD_CLOSURE
  296. BUILD_TUPLE_1
  297. LOAD_LISTCOMP
  298. LOAD_STR MAKE_FUNCTION_CLOSURE
  299. get_aiter CALL_FUNCTION_1
  300. await
  301. set_comp_async ::= LOAD_SETCOMP
  302. LOAD_STR
  303. MAKE_FUNCTION_0
  304. get_aiter
  305. CALL_FUNCTION_1
  306. set_comp_async ::= LOAD_CLOSURE
  307. BUILD_TUPLE_1
  308. LOAD_SETCOMP
  309. LOAD_STR MAKE_FUNCTION_CLOSURE
  310. get_aiter CALL_FUNCTION_1
  311. await
  312. """,
  313. nop_func,
  314. )
  315. custom_ops_processed.add(opname)
  316. self.addRule(
  317. """
  318. dict_comp_async ::= BUILD_MAP_0 LOAD_ARG
  319. dict_comp_async
  320. expr ::= dict_comp_async
  321. expr ::= generator_exp_async
  322. expr ::= list_comp_async
  323. expr ::= set_comp_async
  324. func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
  325. DUP_TOP LOAD_GLOBAL COMPARE_OP
  326. POP_JUMP_IF_TRUE
  327. END_FINALLY _come_froms
  328. # async_iter ::= block_break SETUP_EXCEPT GET_ANEXT
  329. LOAD_CONST YIELD_FROM
  330. get_aiter ::= expr GET_AITER
  331. list_afor ::= get_aiter list_afor2
  332. return_expr_lambda ::= list_comp_async
  333. list_afor2 ::= async_iter store list_iter
  334. JUMP_LOOP COME_FROM_EXCEPT
  335. END_ASYNC_FOR
  336. list_iter ::= list_afor
  337. set_afor ::= get_aiter set_afor2
  338. set_iter ::= set_afor
  339. set_comp_async ::= BUILD_SET_0 LOAD_ARG
  340. set_comp_async
  341. """,
  342. nop_func,
  343. )
  344. custom_ops_processed.add(opname)
  345. elif opname == "GET_ANEXT":
  346. self.addRule(
  347. """
  348. expr ::= genexpr_func_async
  349. expr ::= BUILD_MAP_0 genexpr_func_async
  350. expr ::= list_comp_async
  351. dict_comp_async ::= BUILD_MAP_0 genexpr_func_async
  352. async_iter ::= _come_froms
  353. SETUP_EXCEPT
  354. GET_ANEXT
  355. LOAD_CONST
  356. YIELD_FROM
  357. POP_BLOCK
  358. func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT
  359. LOAD_CONST YIELD_FROM
  360. genexpr_func_async ::= LOAD_ARG async_iter
  361. store
  362. comp_iter
  363. JUMP_LOOP
  364. COME_FROM_EXCEPT
  365. END_ASYNC_FOR
  366. genexpr_func_async ::= LOAD_ARG func_async_prefix
  367. store func_async_middle comp_iter
  368. JUMP_LOOP COME_FROM
  369. POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
  370. list_comp_async ::= LOAD_ARG BUILD_LIST_FROM_ARG list_afor2
  371. list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2
  372. set_afor2 ::= async_iter
  373. store
  374. set_iter
  375. JUMP_LOOP
  376. COME_FROM_FINALLY
  377. END_ASYNC_FOR
  378. set_afor2 ::= expr_or_arg
  379. set_iter_async
  380. set_comp_async ::= BUILD_SET_0 set_afor2
  381. set_iter_async ::= async_iter
  382. store
  383. set_iter
  384. JUMP_LOOP
  385. _come_froms
  386. END_ASYNC_FOR
  387. return_expr_lambda ::= genexpr_func_async
  388. LOAD_CONST RETURN_VALUE
  389. RETURN_VALUE_LAMBDA
  390. return_expr_lambda ::= BUILD_SET_0 genexpr_func_async
  391. RETURN_VALUE_LAMBDA
  392. """,
  393. nop_func,
  394. )
  395. custom_ops_processed.add(opname)
  396. elif opname == "GET_AWAITABLE":
  397. rule_str = """
  398. await ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
  399. await_expr ::= expr await
  400. expr ::= await_expr
  401. """
  402. self.add_unique_doc_rules(rule_str, customize)
  403. elif opname == "GET_ITER":
  404. self.addRule(
  405. """
  406. expr ::= get_iter
  407. get_iter ::= expr GET_ITER
  408. """,
  409. nop_func,
  410. )
  411. custom_ops_processed.add(opname)
  412. elif opname == "LOAD_ASSERT":
  413. if "PyPy" in customize:
  414. rules_str = """
  415. stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
  416. """
  417. self.add_unique_doc_rules(rules_str, customize)
  418. elif opname == "LOAD_ATTR":
  419. self.addRule(
  420. """
  421. expr ::= attribute
  422. attribute ::= expr LOAD_ATTR
  423. """,
  424. nop_func,
  425. )
  426. custom_ops_processed.add(opname)
  427. elif opname == "LOAD_CLOSURE":
  428. self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
  429. elif opname == "LOAD_DICTCOMP":
  430. if has_get_iter_call_function1:
  431. rule_pat = "dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
  432. self.add_make_function_rule(rule_pat, opname, token.attr, customize)
  433. pass
  434. custom_ops_processed.add(opname)
  435. elif opname == "LOAD_GENEXPR":
  436. self.addRule("load_genexpr ::= LOAD_GENEXPR", nop_func)
  437. custom_ops_processed.add(opname)
  438. elif opname == "LOAD_LISTCOMP":
  439. self.add_unique_rule(
  440. "expr ::= list_comp", opname, token.attr, customize
  441. )
  442. custom_ops_processed.add(opname)
  443. elif opname == "LOAD_NAME":
  444. if (
  445. token.attr == "__annotations__"
  446. and "SETUP_ANNOTATIONS" in self.seen_ops
  447. ):
  448. token.kind = "LOAD_ANNOTATION"
  449. self.addRule(
  450. """
  451. stmt ::= SETUP_ANNOTATIONS
  452. stmt ::= ann_assign
  453. ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR
  454. """,
  455. nop_func,
  456. )
  457. pass
  458. elif opname == "LOAD_SETCOMP":
  459. # Should this be generalized and put under MAKE_FUNCTION?
  460. if has_get_iter_call_function1:
  461. self.addRule("expr ::= set_comp", nop_func)
  462. rule_pat = "set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
  463. self.add_make_function_rule(rule_pat, opname, token.attr, customize)
  464. pass
  465. custom_ops_processed.add(opname)
  466. elif opname == "LOOKUP_METHOD":
  467. # A PyPy speciality - DRY with parse3
  468. self.addRule(
  469. """
  470. expr ::= attribute
  471. attribute ::= expr LOOKUP_METHOD
  472. """,
  473. nop_func,
  474. )
  475. custom_ops_processed.add(opname)
  476. elif opname == "MAKE_FUNCTION_CLOSURE":
  477. if "LOAD_DICTCOMP" in self.seen_ops:
  478. # Is there something general going on here?
  479. rule = """
  480. dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
  481. MAKE_FUNCTION_CLOSURE expr
  482. GET_ITER CALL_FUNCTION_1
  483. """
  484. self.addRule(rule, nop_func)
  485. elif "LOAD_SETCOMP" in self.seen_ops:
  486. rule = """
  487. set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
  488. MAKE_FUNCTION_CLOSURE expr
  489. GET_ITER CALL_FUNCTION_1
  490. """
  491. self.addRule(rule, nop_func)
  492. elif opname == "MAKE_FUNCTION_CLOSURE_POS":
  493. args_pos, args_kw, annotate_args, closure = token.attr
  494. stack_count = args_pos + args_kw + annotate_args
  495. if closure:
  496. if args_pos:
  497. # This was seen ion line 447 of Python 3.8
  498. # This is needed for Python 3.8 line 447 of site-packages/nltk/tgrep.py
  499. # line 447:
  500. # lambda i: lambda n, m=None, l=None: ...
  501. # which has
  502. # L. 447 0 LOAD_CONST (None, None)
  503. # 2 LOAD_CLOSURE 'i'
  504. # 4 LOAD_CLOSURE 'predicate'
  505. # 6 BUILD_TUPLE_2 2
  506. # 8 LOAD_LAMBDA '<code_object <lambda>>'
  507. # 10 LOAD_STR '_tgrep_relation_action.<locals>.<lambda>.<locals>.<lambda>'
  508. # 12 MAKE_FUNCTION_CLOSURE_POS 'default, closure'
  509. # FIXME: Possibly we need to generalize for more nested lambda's of lambda's?
  510. rule = """
  511. expr ::= lambda_body
  512. lambda_body ::= %s%s%s%s
  513. """ % (
  514. "expr " * stack_count,
  515. "load_closure " * closure,
  516. "BUILD_TUPLE_2 LOAD_LAMBDA LOAD_STR ",
  517. opname,
  518. )
  519. self.add_unique_rule(rule, opname, token.attr, customize)
  520. rule = """
  521. expr ::= lambda_body
  522. lambda_body ::= %s%s%s%s
  523. """ % (
  524. "expr " * stack_count,
  525. "load_closure " * closure,
  526. "LOAD_LAMBDA LOAD_STR ",
  527. opname,
  528. )
  529. else:
  530. rule = """
  531. expr ::= lambda_body
  532. lambda_body ::= %s%s%s""" % (
  533. "load_closure " * closure,
  534. "LOAD_LAMBDA LOAD_STR ",
  535. opname,
  536. )
  537. self.add_unique_rule(rule, opname, token.attr, customize)
  538. elif opname == "SETUP_WITH":
  539. rules_str = """
  540. with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
  541. WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
  542. # Removes POP_BLOCK LOAD_CONST from 3.6-
  543. with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
  544. WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
  545. """
  546. if self.version < (3, 8):
  547. rules_str += """
  548. with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
  549. LOAD_CONST
  550. WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
  551. """
  552. else:
  553. rules_str += """
  554. with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
  555. BEGIN_FINALLY COME_FROM_WITH
  556. WITH_CLEANUP_START WITH_CLEANUP_FINISH
  557. END_FINALLY
  558. """
  559. self.addRule(rules_str, nop_func)
  560. pass
  561. pass
  562. def custom_classfunc_rule(self, opname, token, customize, next_token):
  563. args_pos, args_kw = self.get_pos_kw(token)
  564. # Additional exprs for * and ** args:
  565. # 0 if neither
  566. # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
  567. # 2 for * and ** args (CALL_FUNCTION_VAR_KW).
  568. # Yes, this computation based on instruction name is a little bit hoaky.
  569. nak = (len(opname) - len("CALL_FUNCTION")) // 3
  570. uniq_param = args_kw + args_pos
  571. if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
  572. rule_str = """
  573. await ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
  574. await_expr ::= expr await
  575. expr ::= await_expr
  576. """
  577. self.add_unique_doc_rules(rule_str, customize)
  578. rule = (
  579. "async_call ::= expr "
  580. + ("expr " * args_pos)
  581. + ("kwarg " * args_kw)
  582. + "expr " * nak
  583. + token.kind
  584. + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
  585. )
  586. self.add_unique_rule(rule, token.kind, uniq_param, customize)
  587. self.add_unique_rule(
  588. "expr ::= async_call", token.kind, uniq_param, customize
  589. )
  590. if opname.startswith("CALL_FUNCTION_KW"):
  591. self.addRule("expr ::= call_kw36", nop_func)
  592. values = "expr " * token.attr
  593. rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
  594. self.add_unique_rule(rule, token.kind, token.attr, customize)
  595. elif opname == "CALL_FUNCTION_EX_KW":
  596. # Note that we don't add to customize token.kind here. Instead, the non-terminal
  597. # names call_ex_kw... are is in semantic actions.
  598. self.addRule(
  599. """expr ::= call_ex_kw4
  600. call_ex_kw4 ::= expr
  601. expr
  602. expr
  603. CALL_FUNCTION_EX_KW
  604. """,
  605. nop_func,
  606. )
  607. if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
  608. self.addRule(
  609. """expr ::= call_ex_kw
  610. call_ex_kw ::= expr expr build_map_unpack_with_call
  611. CALL_FUNCTION_EX_KW
  612. """,
  613. nop_func,
  614. )
  615. if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
  616. # FIXME: should this be parameterized by EX value?
  617. self.addRule(
  618. """expr ::= call_ex_kw3
  619. call_ex_kw3 ::= expr
  620. build_tuple_unpack_with_call
  621. expr
  622. CALL_FUNCTION_EX_KW
  623. """,
  624. nop_func,
  625. )
  626. if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
  627. # FIXME: should this be parameterized by EX value?
  628. self.addRule(
  629. """expr ::= call_ex_kw2
  630. call_ex_kw2 ::= expr
  631. build_tuple_unpack_with_call
  632. build_map_unpack_with_call
  633. CALL_FUNCTION_EX_KW
  634. """,
  635. nop_func,
  636. )
  637. elif opname == "CALL_FUNCTION_EX":
  638. self.addRule(
  639. """
  640. expr ::= call_ex
  641. starred ::= expr
  642. call_ex ::= expr starred CALL_FUNCTION_EX
  643. """,
  644. nop_func,
  645. )
  646. if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
  647. self.addRule(
  648. """
  649. expr ::= call_ex_kw
  650. call_ex_kw ::= expr expr
  651. build_map_unpack_with_call CALL_FUNCTION_EX
  652. """,
  653. nop_func,
  654. )
  655. if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
  656. self.addRule(
  657. """
  658. expr ::= call_ex_kw3
  659. call_ex_kw3 ::= expr
  660. build_tuple_unpack_with_call
  661. %s
  662. CALL_FUNCTION_EX
  663. """
  664. % "expr "
  665. * token.attr,
  666. nop_func,
  667. )
  668. pass
  669. # FIXME: Is this right?
  670. self.addRule(
  671. """
  672. expr ::= call_ex_kw4
  673. call_ex_kw4 ::= expr
  674. expr
  675. expr
  676. CALL_FUNCTION_EX
  677. """,
  678. nop_func,
  679. )
  680. pass
  681. else:
  682. Python37BaseParser.custom_classfunc_rule(
  683. self, opname, token, customize, next_token
  684. )