lambda_custom.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775
  1. # Copyright (c) 2020-2024 Rocky Bernstein
  2. #
  3. # This program is free software: you can redistribute it and/or modify
  4. # it under the terms of the GNU General Public License as published by
  5. # the Free Software Foundation, either version 3 of the License, or
  6. # (at your option) any later version.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License
  14. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  15. """
  16. Grammar Customization rules for Python 3.8's Lambda expression grammar.
  17. """
  18. from decompyle3.parsers.p37.base import Python37BaseParser
  19. from decompyle3.parsers.p38.base import Python38BaseParser
  20. from decompyle3.parsers.parse_heads import nop_func
  21. class Python38LambdaCustom(Python38BaseParser):
  22. def __init__(self):
  23. self.new_rules = set()
  24. # Special opcodes we see that trigger adding new grammar rules.
  25. self.seen_ops = frozenset()
  26. # Special opcodes we see that trigger adding new grammar rules.
  27. self.seen_ops_basenames = frozenset()
  28. # Customized grammar rules
  29. self.customized = {}
  30. def customize_grammar_rules_lambda38(self, tokens, customize):
  31. Python38BaseParser.customize_grammar_rules38(self, tokens, customize)
  32. self.check_reduce["call_kw"] = "AST"
  33. # For a rough break out on the first word. This may
  34. # include instructions that don't need customization,
  35. # but we'll do a finer check after the rough breakout.
  36. customize_instruction_basenames = frozenset(
  37. (
  38. "BEFORE",
  39. "BUILD",
  40. "CALL",
  41. "DICT",
  42. "GET",
  43. "FORMAT",
  44. "LIST",
  45. "LOAD",
  46. "MAKE",
  47. "SETUP",
  48. "UNPACK",
  49. )
  50. )
  51. # Opcode names in the custom_ops_processed set have rules that get
  52. # added unconditionally and the rules are constant. So they need to be
  53. # done only once, and if we see the opcode a second time, we don't have
  54. # to consider adding more rules.
  55. #
  56. custom_ops_processed = frozenset()
  57. # A set of instruction operation names that exist in the token stream.
  58. # We use this to customize the grammar that we create.
  59. # 2.6-compatible set comprehensions
  60. self.seen_ops = frozenset([t.kind for t in tokens])
  61. self.seen_op_basenames = frozenset(
  62. [opname[: opname.rfind("_")] for opname in self.seen_ops]
  63. )
  64. custom_ops_processed = {"DICT_MERGE"}
  65. # Loop over instructions adding custom grammar rules based on
  66. # a specific instruction seen.
  67. if "PyPy" in customize:
  68. self.addRule(
  69. """
  70. stmt ::= assign3_pypy
  71. stmt ::= assign2_pypy
  72. assign3_pypy ::= expr expr expr store store store
  73. assign2_pypy ::= expr expr store store
  74. """,
  75. nop_func,
  76. )
  77. n = len(tokens)
  78. # Determine if we have an iteration CALL_FUNCTION_1.
  79. has_get_iter_call_function1 = False
  80. for i, token in enumerate(tokens):
  81. if (
  82. token == "GET_ITER"
  83. and i < n - 2
  84. and self.call_fn_name(tokens[i + 1]) == "CALL_FUNCTION_1"
  85. ):
  86. has_get_iter_call_function1 = True
  87. for i, token in enumerate(tokens):
  88. opname = token.kind
  89. # Do a quick breakout before testing potentially
  90. # each of the dozen or so instructions in "if"/"elif".
  91. if (
  92. opname[: opname.find("_")] not in customize_instruction_basenames
  93. or opname in custom_ops_processed
  94. ):
  95. continue
  96. opname_base = opname[: opname.rfind("_")]
  97. # Do a quick breakout before testing potentially
  98. # each of the dozen or so instructions in "if"/"elif".
  99. if (
  100. opname[: opname.find("_")] not in customize_instruction_basenames
  101. or opname in custom_ops_processed
  102. ):
  103. continue
  104. if opname == "BEFORE_ASYNC_WITH":
  105. rules_str = """
  106. async_with_post ::= COME_FROM_ASYNC_WITH
  107. WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
  108. WITH_CLEANUP_FINISH END_FINALLY
  109. stmt ::= async_with_as_stmt
  110. async_with_as_stmt ::= expr
  111. async_with_pre
  112. store
  113. suite_stmts_opt
  114. POP_BLOCK LOAD_CONST
  115. async_with_post
  116. async_with_stmt ::= expr
  117. async_with_pre
  118. POP_TOP
  119. c_suite_stmts
  120. POP_BLOCK
  121. BEGIN_FINALLY
  122. WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
  123. WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
  124. POP_BLOCK
  125. BEGIN_FINALLY
  126. COME_FROM_ASYNC_WITH
  127. WITH_CLEANUP_START
  128. GET_AWAITABLE
  129. LOAD_CONST
  130. YIELD_FROM
  131. WITH_CLEANUP_FINISH
  132. END_FINALLY
  133. async_with_stmt ::= expr
  134. async_with_pre
  135. POP_TOP
  136. suite_stmts_opt
  137. POP_BLOCK LOAD_CONST
  138. async_with_post
  139. async_with_stmt ::= expr
  140. async_with_pre
  141. POP_TOP
  142. suite_stmts_opt
  143. async_with_post
  144. """
  145. self.addRule(rules_str, nop_func)
  146. elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
  147. if opname == "BUILD_CONST_DICT":
  148. rule = f"""
  149. add_consts ::= ADD_VALUE*
  150. const_list ::= COLLECTION_START add_consts {opname}
  151. dict ::= const_list
  152. expr ::= dict
  153. """
  154. else:
  155. rule = f"""
  156. add_consts ::= ADD_VALUE*
  157. const_list ::= COLLECTION_START add_consts {opname}
  158. expr ::= const_list
  159. """
  160. self.addRule(rule, nop_func)
  161. elif opname_base in (
  162. "BUILD_LIST",
  163. "BUILD_SET",
  164. "BUILD_SET_UNPACK",
  165. "BUILD_TUPLE",
  166. "BUILD_TUPLE_UNPACK",
  167. ):
  168. v = token.attr
  169. is_LOAD_CLOSURE = False
  170. if opname_base == "BUILD_TUPLE":
  171. # If is part of a "load_closure", then it is not part of a
  172. # "list".
  173. is_LOAD_CLOSURE = True
  174. for j in range(v):
  175. if tokens[i - j - 1].kind != "LOAD_CLOSURE":
  176. is_LOAD_CLOSURE = False
  177. break
  178. if is_LOAD_CLOSURE:
  179. rule_str = "load_closure ::= %s%s" % (
  180. ("LOAD_CLOSURE " * v),
  181. opname,
  182. )
  183. self.add_unique_doc_rules(rule_str, customize)
  184. elif opname_base == "BUILD_LIST":
  185. v = token.attr
  186. if v == 0:
  187. rule_str = """
  188. list ::= BUILD_LIST_0
  189. list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND
  190. list ::= list_unpack
  191. """
  192. self.add_unique_doc_rules(rule_str, customize)
  193. elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL":
  194. # FIXME: should this be parameterized by EX value?
  195. self.addRule(
  196. """expr ::= call_ex_kw3
  197. call_ex_kw3 ::= expr
  198. build_tuple_unpack_with_call
  199. expr
  200. CALL_FUNCTION_EX_KW
  201. """,
  202. nop_func,
  203. )
  204. if not is_LOAD_CLOSURE or v == 0:
  205. # We do this complicated test to speed up parsing of
  206. # pathologically long literals, especially those over 1024.
  207. build_count = token.attr
  208. thousands = build_count // 1024
  209. thirty32s = (build_count // 32) % 32
  210. if thirty32s > 0:
  211. rule = "expr32 ::=%s" % (" expr" * 32)
  212. self.add_unique_rule(rule, opname_base, build_count, customize)
  213. pass
  214. if thousands > 0:
  215. self.add_unique_rule(
  216. "expr1024 ::=%s" % (" expr32" * 32),
  217. opname_base,
  218. build_count,
  219. customize,
  220. )
  221. pass
  222. collection = opname_base[opname_base.find("_") + 1 :].lower()
  223. rule = (
  224. ("%s ::= " % collection)
  225. + "expr1024 " * thousands
  226. + "expr32 " * thirty32s
  227. + "expr " * (build_count % 32)
  228. + opname
  229. )
  230. self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
  231. continue
  232. continue
  233. elif opname.startswith("BUILD_STRING"):
  234. v = token.attr
  235. rules_str = """
  236. expr ::= joined_str
  237. joined_str ::= %sBUILD_STRING_%d
  238. """ % (
  239. "expr " * v,
  240. v,
  241. )
  242. self.add_unique_doc_rules(rules_str, customize)
  243. if "FORMAT_VALUE_ATTR" in self.seen_ops:
  244. rules_str = """
  245. formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
  246. expr ::= formatted_value_attr
  247. """
  248. self.add_unique_doc_rules(rules_str, customize)
  249. elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
  250. v = token.attr
  251. rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
  252. self.addRule(rule, nop_func)
  253. elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
  254. v = token.attr
  255. rule = (
  256. "build_tuple_unpack_with_call ::= "
  257. + "expr1024 " * int(v // 1024)
  258. + "expr32 " * int((v // 32) % 32)
  259. + "expr " * (v % 32)
  260. + opname
  261. )
  262. self.addRule(rule, nop_func)
  263. rule = "starred ::= %s %s" % ("expr " * v, opname)
  264. self.addRule(rule, nop_func)
  265. elif opname == "FORMAT_VALUE":
  266. rules_str = """
  267. expr ::= formatted_value1
  268. formatted_value1 ::= expr FORMAT_VALUE
  269. """
  270. self.add_unique_doc_rules(rules_str, customize)
  271. elif opname == "FORMAT_VALUE_ATTR":
  272. rules_str = """
  273. expr ::= formatted_value2
  274. formatted_value2 ::= expr expr FORMAT_VALUE_ATTR
  275. """
  276. self.add_unique_doc_rules(rules_str, customize)
  277. elif opname == "GET_AITER":
  278. self.add_unique_doc_rules("get_aiter ::= expr GET_AITER", customize)
  279. if {"MAKE_FUNCTION_0", "MAKE_FUNCTION_CLOSURE"} not in self.seen_ops:
  280. self.addRule(
  281. """
  282. expr ::= dict_comp_async
  283. expr ::= generator_exp_async
  284. expr ::= list_comp_async
  285. dict_comp_async ::= LOAD_DICTCOMP
  286. LOAD_STR
  287. MAKE_FUNCTION_0
  288. get_aiter
  289. CALL_FUNCTION_1
  290. dict_comp_async ::= BUILD_MAP_0 LOAD_ARG
  291. dict_comp_async
  292. generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0
  293. get_aiter CALL_FUNCTION_1
  294. list_comp_async ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0
  295. get_aiter CALL_FUNCTION_1
  296. await
  297. list_comp_async ::= LOAD_CLOSURE
  298. BUILD_TUPLE_1
  299. LOAD_LISTCOMP
  300. LOAD_STR MAKE_FUNCTION_CLOSURE
  301. get_aiter CALL_FUNCTION_1
  302. await
  303. set_comp_async ::= LOAD_SETCOMP
  304. LOAD_STR
  305. MAKE_FUNCTION_0
  306. get_aiter
  307. CALL_FUNCTION_1
  308. set_comp_async ::= LOAD_CLOSURE
  309. BUILD_TUPLE_1
  310. LOAD_SETCOMP
  311. LOAD_STR MAKE_FUNCTION_CLOSURE
  312. get_aiter CALL_FUNCTION_1
  313. await
  314. """,
  315. nop_func,
  316. )
  317. custom_ops_processed.add(opname)
  318. self.addRule(
  319. """
  320. dict_comp_async ::= BUILD_MAP_0 LOAD_ARG
  321. dict_comp_async
  322. expr ::= dict_comp_async
  323. expr ::= generator_exp_async
  324. expr ::= list_comp_async
  325. expr ::= set_comp_async
  326. func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
  327. DUP_TOP LOAD_GLOBAL COMPARE_OP
  328. POP_JUMP_IF_TRUE
  329. END_FINALLY _come_froms
  330. # async_iter ::= block_break SETUP_EXCEPT GET_ANEXT
  331. LOAD_CONST YIELD_FROM
  332. get_aiter ::= expr GET_AITER
  333. list_afor ::= get_aiter list_afor2
  334. list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2
  335. list_iter ::= list_afor
  336. set_afor ::= get_aiter set_afor2
  337. set_iter ::= set_afor
  338. set_comp_async ::= BUILD_SET_0 LOAD_ARG
  339. set_comp_async
  340. """,
  341. nop_func,
  342. )
  343. custom_ops_processed.add(opname)
  344. elif opname == "GET_ANEXT":
  345. self.addRule(
  346. """
  347. expr ::= genexpr_func_async
  348. expr ::= BUILD_MAP_0 genexpr_func_async
  349. expr ::= list_comp_async
  350. dict_comp_async ::= BUILD_MAP_0 genexpr_func_async
  351. async_iter ::= _come_froms
  352. SETUP_FINALLY GET_ANEXT LOAD_CONST
  353. YIELD_FROM POP_BLOCK
  354. func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT
  355. LOAD_CONST YIELD_FROM
  356. genexpr_func_async ::= LOAD_ARG async_iter
  357. store
  358. comp_iter
  359. JUMP_LOOP
  360. COME_FROM_FINALLY
  361. END_ASYNC_FOR
  362. genexpr_func_async ::= LOAD_ARG func_async_prefix
  363. store func_async_middle comp_iter
  364. JUMP_LOOP COME_FROM
  365. POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
  366. list_afor2 ::= async_iter
  367. store
  368. list_iter
  369. JUMP_LOOP
  370. COME_FROM_FINALLY
  371. END_ASYNC_FOR
  372. list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2
  373. set_afor2 ::= async_iter
  374. store
  375. set_iter
  376. JUMP_LOOP
  377. COME_FROM_FINALLY
  378. END_ASYNC_FOR
  379. set_afor2 ::= expr_or_arg
  380. set_iter_async
  381. set_comp_async ::= BUILD_SET_0 set_afor2
  382. set_iter_async ::= async_iter
  383. store
  384. set_iter
  385. JUMP_LOOP
  386. _come_froms
  387. END_ASYNC_FOR
  388. return_expr_lambda ::= genexpr_func_async
  389. LOAD_CONST RETURN_VALUE
  390. RETURN_VALUE_LAMBDA
  391. return_expr_lambda ::= BUILD_SET_0 genexpr_func_async
  392. RETURN_VALUE_LAMBDA
  393. """,
  394. nop_func,
  395. )
  396. custom_ops_processed.add(opname)
  397. elif opname == "GET_AWAITABLE":
  398. rule_str = """
  399. await ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
  400. await_expr ::= expr await
  401. expr ::= await_expr
  402. """
  403. self.add_unique_doc_rules(rule_str, customize)
  404. elif opname == "GET_ITER":
  405. self.addRule(
  406. """
  407. expr ::= get_iter
  408. get_iter ::= expr GET_ITER
  409. """,
  410. nop_func,
  411. )
  412. custom_ops_processed.add(opname)
  413. elif opname == "LOAD_ASSERT":
  414. if "PyPy" in customize:
  415. rules_str = """
  416. stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
  417. """
  418. self.add_unique_doc_rules(rules_str, customize)
  419. elif opname == "LOAD_ATTR":
  420. self.addRule(
  421. """
  422. expr ::= attribute
  423. attribute ::= expr LOAD_ATTR
  424. """,
  425. nop_func,
  426. )
  427. custom_ops_processed.add(opname)
  428. elif opname == "LOAD_CLOSURE":
  429. self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
  430. elif opname == "LOAD_DICTCOMP":
  431. if has_get_iter_call_function1:
  432. rule_pat = "dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
  433. self.add_make_function_rule(rule_pat, opname, token.attr, customize)
  434. pass
  435. custom_ops_processed.add(opname)
  436. elif opname == "LOAD_GENEXPR":
  437. self.addRule("load_genexpr ::= LOAD_GENEXPR", nop_func)
  438. custom_ops_processed.add(opname)
  439. elif opname == "LOAD_LISTCOMP":
  440. self.add_unique_rule(
  441. "expr ::= list_comp", opname, token.attr, customize
  442. )
  443. custom_ops_processed.add(opname)
  444. elif opname == "LOAD_NAME":
  445. if (
  446. token.attr == "__annotations__"
  447. and "SETUP_ANNOTATIONS" in self.seen_ops
  448. ):
  449. token.kind = "LOAD_ANNOTATION"
  450. self.addRule(
  451. """
  452. stmt ::= SETUP_ANNOTATIONS
  453. stmt ::= ann_assign
  454. ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR
  455. """,
  456. nop_func,
  457. )
  458. pass
  459. elif opname == "LOAD_SETCOMP":
  460. # Should this be generalized and put under MAKE_FUNCTION?
  461. if has_get_iter_call_function1:
  462. self.addRule("expr ::= set_comp", nop_func)
  463. rule_pat = "set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
  464. self.add_make_function_rule(rule_pat, opname, token.attr, customize)
  465. pass
  466. custom_ops_processed.add(opname)
  467. elif opname == "LOOKUP_METHOD":
  468. # A PyPy speciality - DRY with parse3
  469. self.addRule(
  470. """
  471. expr ::= attribute
  472. attribute ::= expr LOOKUP_METHOD
  473. """,
  474. nop_func,
  475. )
  476. custom_ops_processed.add(opname)
  477. elif opname == "MAKE_FUNCTION_CLOSURE":
  478. if "LOAD_DICTCOMP" in self.seen_ops:
  479. # Is there something general going on here?
  480. rule = """
  481. dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
  482. MAKE_FUNCTION_CLOSURE expr
  483. GET_ITER CALL_FUNCTION_1
  484. """
  485. self.addRule(rule, nop_func)
  486. elif "LOAD_SETCOMP" in self.seen_ops:
  487. rule = """
  488. set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
  489. MAKE_FUNCTION_CLOSURE expr
  490. GET_ITER CALL_FUNCTION_1
  491. """
  492. self.addRule(rule, nop_func)
  493. elif opname == "MAKE_FUNCTION_CLOSURE_POS":
  494. args_pos, args_kw, annotate_args, closure = token.attr
  495. stack_count = args_pos + args_kw + annotate_args
  496. if closure:
  497. if args_pos:
  498. # This was seen ion line 447 of Python 3.8
  499. # This is needed for Python 3.8 line 447 of site-packages/nltk/tgrep.py
  500. # line 447:
  501. # lambda i: lambda n, m=None, l=None: ...
  502. # which has
  503. # L. 447 0 LOAD_CONST (None, None)
  504. # 2 LOAD_CLOSURE 'i'
  505. # 4 LOAD_CLOSURE 'predicate'
  506. # 6 BUILD_TUPLE_2 2
  507. # 8 LOAD_LAMBDA '<code_object <lambda>>'
  508. # 10 LOAD_STR '_tgrep_relation_action.<locals>...'
  509. # 12 MAKE_FUNCTION_CLOSURE_POS 'default, closure'
  510. # FIXME: Possibly we need to generalize for more nested lambda's of lambda's?
  511. rule = """
  512. expr ::= lambda_body
  513. lambda_body ::= %s%s%s%s
  514. """ % (
  515. "expr " * stack_count,
  516. "load_closure " * closure,
  517. "BUILD_TUPLE_2 LOAD_LAMBDA LOAD_STR ",
  518. opname,
  519. )
  520. self.add_unique_rule(rule, opname, token.attr, customize)
  521. rule = """
  522. expr ::= lambda_body
  523. lambda_body ::= %s%s%s%s
  524. """ % (
  525. "expr " * stack_count,
  526. "load_closure " * closure,
  527. "LOAD_LAMBDA LOAD_STR ",
  528. opname,
  529. )
  530. else:
  531. rule = """
  532. expr ::= lambda_body
  533. lambda_body ::= %s%s%s""" % (
  534. "load_closure " * closure,
  535. "LOAD_LAMBDA LOAD_STR ",
  536. opname,
  537. )
  538. self.add_unique_rule(rule, opname, token.attr, customize)
  539. elif opname == "SETUP_WITH":
  540. rules_str = """
  541. with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
  542. WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
  543. # Removes POP_BLOCK LOAD_CONST from 3.6-
  544. with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
  545. a WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
  546. """
  547. if self.version < (3, 8):
  548. rules_str += """
  549. with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
  550. LOAD_CONST
  551. WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
  552. """
  553. else:
  554. rules_str += """
  555. with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
  556. BEGIN_FINALLY COME_FROM_WITH
  557. WITH_CLEANUP_START WITH_CLEANUP_FINISH
  558. END_FINALLY
  559. """
  560. self.addRule(rules_str, nop_func)
  561. pass
  562. pass
  563. def custom_classfunc_rule(self, opname, token, customize, next_token):
  564. args_pos, args_kw = self.get_pos_kw(token)
  565. # Additional exprs for * and ** args:
  566. # 0 if neither
  567. # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
  568. # 2 for * and ** args (CALL_FUNCTION_VAR_KW).
  569. # Yes, this computation based on instruction name is a little bit hoaky.
  570. nak = (len(opname) - len("CALL_FUNCTION")) // 3
  571. uniq_param = args_kw + args_pos
  572. if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
  573. rule_str = """
  574. await ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
  575. await_expr ::= expr await
  576. expr ::= await_expr
  577. """
  578. self.add_unique_doc_rules(rule_str, customize)
  579. rule = (
  580. "async_call ::= expr "
  581. + ("expr " * args_pos)
  582. + ("kwarg " * args_kw)
  583. + "expr " * nak
  584. + token.kind
  585. + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
  586. )
  587. self.add_unique_rule(rule, token.kind, uniq_param, customize)
  588. self.add_unique_rule(
  589. "expr ::= async_call", token.kind, uniq_param, customize
  590. )
  591. if opname.startswith("CALL_FUNCTION_KW"):
  592. self.addRule("expr ::= call_kw36", nop_func)
  593. values = "expr " * token.attr
  594. rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
  595. self.add_unique_rule(rule, token.kind, token.attr, customize)
  596. elif opname == "CALL_FUNCTION_EX_KW":
  597. # Note that we don't add to customize token.kind here.
  598. # Instead, the non-terminal names, "call_ex_kw"...
  599. # are in semantic actions.
  600. self.addRule(
  601. """expr ::= call_ex_kw4
  602. call_ex_kw4 ::= expr
  603. expr
  604. expr
  605. CALL_FUNCTION_EX_KW
  606. """,
  607. nop_func,
  608. )
  609. if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
  610. self.addRule(
  611. """expr ::= call_ex_kw
  612. call_ex_kw ::= expr expr build_map_unpack_with_call
  613. CALL_FUNCTION_EX_KW
  614. """,
  615. nop_func,
  616. )
  617. if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
  618. # FIXME: should this be parameterized by EX value?
  619. self.addRule(
  620. """expr ::= call_ex_kw3
  621. call_ex_kw3 ::= expr
  622. build_tuple_unpack_with_call
  623. expr
  624. CALL_FUNCTION_EX_KW
  625. """,
  626. nop_func,
  627. )
  628. if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
  629. # FIXME: should this be parameterized by EX value?
  630. self.addRule(
  631. """expr ::= call_ex_kw2
  632. call_ex_kw2 ::= expr
  633. build_tuple_unpack_with_call
  634. build_map_unpack_with_call
  635. CALL_FUNCTION_EX_KW
  636. """,
  637. nop_func,
  638. )
  639. elif opname == "CALL_FUNCTION_EX":
  640. self.addRule(
  641. """
  642. expr ::= call_ex
  643. starred ::= expr
  644. call_ex ::= expr starred CALL_FUNCTION_EX
  645. """,
  646. nop_func,
  647. )
  648. if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
  649. self.addRule(
  650. """
  651. expr ::= call_ex_kw
  652. call_ex_kw ::= expr expr
  653. build_map_unpack_with_call CALL_FUNCTION_EX
  654. """,
  655. nop_func,
  656. )
  657. if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
  658. self.addRule(
  659. """
  660. expr ::= call_ex_kw3
  661. call_ex_kw3 ::= expr
  662. build_tuple_unpack_with_call
  663. %s
  664. CALL_FUNCTION_EX
  665. """
  666. % "expr "
  667. * token.attr,
  668. nop_func,
  669. )
  670. pass
  671. # FIXME: Is this right?
  672. self.addRule(
  673. """
  674. expr ::= call_ex_kw4
  675. call_ex_kw4 ::= expr
  676. expr
  677. expr
  678. CALL_FUNCTION_EX
  679. """,
  680. nop_func,
  681. )
  682. pass
  683. else:
  684. Python37BaseParser.custom_classfunc_rule(
  685. self, opname, token, customize, next_token
  686. )