_position_node_finder.py 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045
  1. import ast
  2. import sys
  3. import dis
  4. from types import CodeType, FrameType
  5. from typing import Any, Callable, Iterator, Optional, Sequence, Set, Tuple, Type, Union, cast
  6. from .executing import EnhancedAST, NotOneValueFound, Source, only, function_node_types, assert_
  7. from ._exceptions import KnownIssue, VerifierFailure
  8. from ._utils import mangled_name
  9. from functools import lru_cache
  10. import itertools
  11. # the code in this module can use all python>=3.11 features
  12. def parents(node: EnhancedAST) -> Iterator[EnhancedAST]:
  13. while True:
  14. if hasattr(node, "parent"):
  15. node = node.parent
  16. yield node
  17. else:
  18. break # pragma: no mutate
  19. def node_and_parents(node: EnhancedAST) -> Iterator[EnhancedAST]:
  20. yield node
  21. yield from parents(node)
  22. @lru_cache(128) # pragma: no mutate
  23. def get_instructions(code: CodeType) -> list[dis.Instruction]:
  24. return list(dis.get_instructions(code))
  25. types_cmp_issue_fix = (
  26. ast.IfExp,
  27. ast.If,
  28. ast.Assert,
  29. ast.While,
  30. )
  31. types_cmp_issue = types_cmp_issue_fix + (
  32. ast.ListComp,
  33. ast.SetComp,
  34. ast.DictComp,
  35. ast.GeneratorExp,
  36. )
  37. op_type_map = {
  38. "**": ast.Pow,
  39. "*": ast.Mult,
  40. "@": ast.MatMult,
  41. "//": ast.FloorDiv,
  42. "/": ast.Div,
  43. "%": ast.Mod,
  44. "+": ast.Add,
  45. "-": ast.Sub,
  46. "<<": ast.LShift,
  47. ">>": ast.RShift,
  48. "&": ast.BitAnd,
  49. "^": ast.BitXor,
  50. "|": ast.BitOr,
  51. }
  52. class PositionNodeFinder(object):
  53. """
  54. Mapping bytecode to ast-node based on the source positions, which where introduced in pyhon 3.11.
  55. In general every ast-node can be exactly referenced by its begin/end line/col_offset, which is stored in the bytecode.
  56. There are only some exceptions for methods and attributes.
  57. """
  58. def __init__(self, frame: FrameType, stmts: Set[EnhancedAST], tree: ast.Module, lasti: int, source: Source):
  59. self.bc_dict={bc.offset:bc for bc in get_instructions(frame.f_code) }
  60. self.frame=frame
  61. self.source = source
  62. self.decorator: Optional[EnhancedAST] = None
  63. # work around for https://github.com/python/cpython/issues/96970
  64. while self.opname(lasti) == "CACHE":
  65. lasti -= 2
  66. try:
  67. # try to map with all match_positions
  68. self.result = self.find_node(lasti)
  69. except NotOneValueFound:
  70. typ: tuple[Type]
  71. # LOAD_METHOD could load "".join for long "..."%(...) BinOps
  72. # this can only be associated by using all positions
  73. if self.opname(lasti) in (
  74. "LOAD_METHOD",
  75. "LOAD_ATTR",
  76. "STORE_ATTR",
  77. "DELETE_ATTR",
  78. ):
  79. # lineno and col_offset of LOAD_METHOD and *_ATTR instructions get set to the beginning of
  80. # the attribute by the python compiler to improved error messages (PEP-657)
  81. # we ignore here the start position and try to find the ast-node just by end position and expected node type
  82. # This is save, because there can only be one attribute ending at a specific point in the source code.
  83. typ = (ast.Attribute,)
  84. elif self.opname(lasti) in ("CALL", "CALL_KW"):
  85. # A CALL instruction can be a method call, in which case the lineno and col_offset gets changed by the compiler.
  86. # Therefore we ignoring here this attributes and searchnig for a Call-node only by end_col_offset and end_lineno.
  87. # This is save, because there can only be one method ending at a specific point in the source code.
  88. # One closing ) only belongs to one method.
  89. typ = (ast.Call,)
  90. else:
  91. raise
  92. self.result = self.find_node(
  93. lasti,
  94. match_positions=("end_col_offset", "end_lineno"),
  95. typ=typ,
  96. )
  97. instruction = self.instruction(lasti)
  98. assert instruction is not None
  99. self.result = self.fix_result(self.result, instruction)
  100. self.known_issues(self.result, instruction)
  101. self.test_for_decorator(self.result, lasti)
  102. # verify
  103. if self.decorator is None:
  104. self.verify(self.result, instruction)
  105. else:
  106. assert_(self.decorator in self.result.decorator_list)
  107. def test_for_decorator(self, node: EnhancedAST, index: int) -> None:
  108. if (
  109. isinstance(node.parent, (ast.ClassDef, function_node_types))
  110. and node in node.parent.decorator_list # type: ignore[attr-defined]
  111. ):
  112. node_func = node.parent
  113. while True:
  114. # the generated bytecode looks like follow:
  115. # index opname
  116. # ------------------
  117. # index-4 PRECALL (only in 3.11)
  118. # index-2 CACHE
  119. # index CALL <- the call instruction
  120. # ... CACHE some CACHE instructions
  121. # maybe multiple other bytecode blocks for other decorators
  122. # index-4 PRECALL (only in 3.11)
  123. # index-2 CACHE
  124. # index CALL <- index of the next loop
  125. # ... CACHE some CACHE instructions
  126. # index+x STORE_* the ast-node of this instruction points to the decorated thing
  127. if not (
  128. (self.opname(index - 4) == "PRECALL" or sys.version_info >= (3, 12))
  129. and self.opname(index) == "CALL"
  130. ): # pragma: no mutate
  131. break # pragma: no mutate
  132. index += 2
  133. while self.opname(index) in ("CACHE", "EXTENDED_ARG"):
  134. index += 2
  135. if (
  136. self.opname(index).startswith("STORE_")
  137. and self.find_node(index) == node_func
  138. ):
  139. self.result = node_func
  140. self.decorator = node
  141. return
  142. if sys.version_info < (3, 12):
  143. index += 4
  144. def fix_result(
  145. self, node: EnhancedAST, instruction: dis.Instruction
  146. ) -> EnhancedAST:
  147. if (
  148. sys.version_info >= (3, 12, 5)
  149. and instruction.opname in ("GET_ITER", "FOR_ITER")
  150. and isinstance(node.parent, ast.For)
  151. and node is node.parent.iter
  152. ):
  153. # node positions have changed in 3.12.5
  154. # https://github.com/python/cpython/issues/93691
  155. # `for` calls __iter__ and __next__ during execution, the calling
  156. # expression of these calls was the ast.For node since cpython 3.11 (see test_iter).
  157. # cpython 3.12.5 changed this to the `iter` node of the loop, to make tracebacks easier to read.
  158. # This keeps backward compatibility with older executing versions.
  159. # there are also cases like:
  160. #
  161. # for a in iter(l): pass
  162. #
  163. # where `iter(l)` would be otherwise the resulting node for the `iter()` call and the __iter__ call of the for implementation.
  164. # keeping the old behaviour makes it possible to distinguish both cases.
  165. return node.parent
  166. if (
  167. sys.version_info >= (3, 12, 6)
  168. and instruction.opname in ("GET_ITER", "FOR_ITER")
  169. and isinstance(
  170. node.parent.parent,
  171. (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp),
  172. )
  173. and isinstance(node.parent,ast.comprehension)
  174. and node is node.parent.iter
  175. ):
  176. # same as above but only for comprehensions, see:
  177. # https://github.com/python/cpython/issues/123142
  178. return node.parent.parent
  179. if sys.version_info >= (3, 12,6) and instruction.opname == "CALL":
  180. before = self.instruction_before(instruction)
  181. if (
  182. before is not None
  183. and before.opname == "LOAD_CONST"
  184. and before.positions == instruction.positions
  185. and isinstance(node.parent, ast.withitem)
  186. and node is node.parent.context_expr
  187. ):
  188. # node positions for with-statements have change
  189. # and is now equal to the expression which created the context-manager
  190. # https://github.com/python/cpython/pull/120763
  191. # with context_manager:
  192. # ...
  193. # but there is one problem to distinguish call-expressions from __exit__()
  194. # with context_manager():
  195. # ...
  196. # the call for __exit__
  197. # 20 1:5 1:22 LOAD_CONST(None)
  198. # 22 1:5 1:22 LOAD_CONST(None)
  199. # 24 1:5 1:22 LOAD_CONST(None)
  200. # 26 1:5 1:22 CALL() # <-- same source range as context_manager()
  201. # but we can use the fact that the previous load for None
  202. # has the same source range as the call, wich can not happen for normal calls
  203. # we return the same ast.With statement at the and to preserve backward compatibility
  204. return node.parent.parent
  205. if (
  206. sys.version_info >= (3, 12,6)
  207. and instruction.opname == "BEFORE_WITH"
  208. and isinstance(node.parent, ast.withitem)
  209. and node is node.parent.context_expr
  210. ):
  211. # handle positions changes for __enter__
  212. return node.parent.parent
  213. if sys.version_info >= (3, 14) and instruction.opname == "CALL":
  214. before = self.instruction_before(instruction)
  215. if (
  216. before is not None
  217. and before.opname == "LOAD_SPECIAL"
  218. and before.argrepr in ("__enter__","__aenter__")
  219. and before.positions == instruction.positions
  220. and isinstance(node.parent, ast.withitem)
  221. and node is node.parent.context_expr
  222. ):
  223. return node.parent.parent
  224. if sys.version_info >= (3, 14) and isinstance(node, ast.UnaryOp) and isinstance(node.op,ast.Not) and instruction.opname !="UNARY_NOT":
  225. # fix for https://github.com/python/cpython/issues/137843
  226. return node.operand
  227. return node
  228. def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None:
  229. if instruction.opname in ("COMPARE_OP", "IS_OP", "CONTAINS_OP") and isinstance(
  230. node, types_cmp_issue
  231. ):
  232. if isinstance(node, types_cmp_issue_fix):
  233. # this is a workaround for https://github.com/python/cpython/issues/95921
  234. # we can fix cases with only on comparison inside the test condition
  235. #
  236. # we can not fix cases like:
  237. # if a<b<c and d<e<f: pass
  238. # if (a<b<c)!=d!=e: pass
  239. # because we don't know which comparison caused the problem
  240. comparisons = [
  241. n
  242. for n in ast.walk(node.test) # type: ignore[attr-defined]
  243. if isinstance(n, ast.Compare) and len(n.ops) > 1
  244. ]
  245. assert_(comparisons, "expected at least one comparison")
  246. if len(comparisons) == 1:
  247. node = self.result = cast(EnhancedAST, comparisons[0])
  248. else:
  249. raise KnownIssue(
  250. "multiple chain comparison inside %s can not be fixed" % (node)
  251. )
  252. else:
  253. # Comprehension and generators get not fixed for now.
  254. raise KnownIssue("chain comparison inside %s can not be fixed" % (node))
  255. if (
  256. sys.version_info[:3] == (3, 11, 1)
  257. and isinstance(node, ast.Compare)
  258. and instruction.opname == "CALL"
  259. and any(isinstance(n, ast.Assert) for n in node_and_parents(node))
  260. ):
  261. raise KnownIssue(
  262. "known bug in 3.11.1 https://github.com/python/cpython/issues/95921"
  263. )
  264. if isinstance(node, ast.Assert):
  265. # pytest assigns the position of the assertion to all expressions of the rewritten assertion.
  266. # All the rewritten expressions get mapped to ast.Assert, which is the wrong ast-node.
  267. # We don't report this wrong result.
  268. raise KnownIssue("assert")
  269. if any(isinstance(n, ast.pattern) for n in node_and_parents(node)):
  270. # TODO: investigate
  271. raise KnownIssue("pattern matching ranges seems to be wrong")
  272. if (
  273. sys.version_info >= (3, 12)
  274. and isinstance(node, ast.Call)
  275. and isinstance(node.func, ast.Name)
  276. and node.func.id == "super"
  277. ):
  278. # super is optimized to some instructions which do not map nicely to a Call
  279. # find the enclosing function
  280. func = node.parent
  281. while hasattr(func, "parent") and not isinstance(
  282. func, (ast.AsyncFunctionDef, ast.FunctionDef)
  283. ):
  284. func = func.parent
  285. # get the first function argument (self/cls)
  286. first_arg = None
  287. if hasattr(func, "args"):
  288. args = [*func.args.posonlyargs, *func.args.args]
  289. if args:
  290. first_arg = args[0].arg
  291. if (instruction.opname, instruction.argval) in [
  292. ("LOAD_DEREF", "__class__"),
  293. ("LOAD_FAST", first_arg),
  294. ("LOAD_FAST_BORROW", first_arg),
  295. ("LOAD_DEREF", first_arg),
  296. ]:
  297. raise KnownIssue("super optimization")
  298. if self.is_except_cleanup(instruction, node):
  299. raise KnownIssue("exeption cleanup does not belong to the last node in a except block")
  300. if instruction.opname == "STORE_NAME" and instruction.argval == "__classcell__":
  301. # handle stores to __classcell__ as KnownIssue,
  302. # because they get complicated if they are used in `if` or `for` loops
  303. # example:
  304. #
  305. # class X:
  306. # # ... something
  307. # if some_condition:
  308. # def method(self):
  309. # super()
  310. #
  311. # The `STORE_NAME` instruction gets mapped to the `ast.If` node,
  312. # because it is the last element in the class.
  313. # This last element could be anything and gets dificult to verify.
  314. raise KnownIssue("store __classcell__")
  315. if (
  316. instruction.opname == "CALL"
  317. and not isinstance(node,ast.Call)
  318. and any(isinstance(p, ast.Assert) for p in parents(node))
  319. and sys.version_info >= (3, 11, 2)
  320. ):
  321. raise KnownIssue("exception generation maps to condition")
  322. if sys.version_info >= (3, 13):
  323. if instruction.opname in (
  324. "STORE_FAST_STORE_FAST",
  325. "STORE_FAST_LOAD_FAST",
  326. "LOAD_FAST_LOAD_FAST",
  327. ):
  328. raise KnownIssue(f"can not map {instruction.opname} to two ast nodes")
  329. if instruction.opname in ("LOAD_FAST","LOAD_FAST_BORROW") and instruction.argval == "__class__":
  330. # example:
  331. # class T:
  332. # def a():
  333. # super()
  334. # some_node # <- there is a LOAD_FAST for this node because we use super()
  335. raise KnownIssue(
  336. f"loading of __class__ is accociated with a random node at the end of a class if you use super()"
  337. )
  338. if (
  339. instruction.opname == "COMPARE_OP"
  340. and isinstance(node, ast.UnaryOp)
  341. and isinstance(node.operand,ast.Compare)
  342. and isinstance(node.op, ast.Not)
  343. ):
  344. # work around for
  345. # https://github.com/python/cpython/issues/114671
  346. self.result = node.operand
  347. if sys.version_info >= (3,14):
  348. if header_length := self.annotation_header_size():
  349. last_offset=list(self.bc_dict.keys())[-1]
  350. if (
  351. not (header_length*2 < instruction.offset <last_offset-4)
  352. ):
  353. # https://github.com/python/cpython/issues/135700
  354. raise KnownIssue("synthetic opcodes in annotations are just bound to the first node")
  355. if self.frame.f_code.co_name=="__annotate__" and instruction.opname=="STORE_SUBSCR":
  356. raise KnownIssue("synthetic code to store annotation")
  357. if self.frame.f_code.co_name=="__annotate__" and isinstance(node,ast.AnnAssign):
  358. raise KnownIssue("some opcodes in the annotation are just bound specific nodes")
  359. if isinstance(node,(ast.TypeAlias)) and self.frame.f_code.co_name==node.name.id :
  360. raise KnownIssue("some opcodes in the annotation are just bound TypeAlias")
  361. if instruction.opname == "STORE_NAME" and instruction.argrepr == "__annotate__":
  362. raise KnownIssue("just a store of the annotation")
  363. if instruction.opname == "IS_OP" and isinstance(node,ast.Name):
  364. raise KnownIssue("part of a check that a name like `all` is a builtin")
  365. def annotation_header_size(self)->int:
  366. if sys.version_info >=(3,14):
  367. header=[inst.opname for inst in itertools.islice(self.bc_dict.values(),8)]
  368. if len(header)==8:
  369. if header[0] in ("COPY_FREE_VARS","MAKE_CELL"):
  370. del header[0]
  371. header_size=8
  372. else:
  373. del header[7]
  374. header_size=7
  375. if header==[
  376. "RESUME",
  377. "LOAD_FAST_BORROW",
  378. "LOAD_SMALL_INT",
  379. "COMPARE_OP",
  380. "POP_JUMP_IF_FALSE",
  381. "NOT_TAKEN",
  382. "LOAD_COMMON_CONSTANT",
  383. ]:
  384. return header_size
  385. return 0
  386. @staticmethod
  387. def is_except_cleanup(inst: dis.Instruction, node: EnhancedAST) -> bool:
  388. if inst.opname not in (
  389. "STORE_NAME",
  390. "STORE_FAST",
  391. "STORE_DEREF",
  392. "STORE_GLOBAL",
  393. "DELETE_NAME",
  394. "DELETE_FAST",
  395. "DELETE_DEREF",
  396. "DELETE_GLOBAL",
  397. ):
  398. return False
  399. # This bytecode does something exception cleanup related.
  400. # The position of the instruciton seems to be something in the last ast-node of the ExceptHandler
  401. # this could be a bug, but it might not be observable in normal python code.
  402. # example:
  403. # except Exception as exc:
  404. # enum_member._value_ = value
  405. # other example:
  406. # STORE_FAST of e was mapped to Constant(value=False)
  407. # except OSError as e:
  408. # if not _ignore_error(e):
  409. # raise
  410. # return False
  411. # STORE_FAST of msg was mapped to print(...)
  412. # except TypeError as msg:
  413. # print("Sorry:", msg, file=file)
  414. if (
  415. isinstance(node, ast.Name)
  416. and isinstance(node.ctx,ast.Store)
  417. and inst.opname.startswith("STORE_")
  418. and mangled_name(node) == inst.argval
  419. ):
  420. # Storing the variable is valid and no exception cleanup, if the name is correct
  421. return False
  422. if (
  423. isinstance(node, ast.Name)
  424. and isinstance(node.ctx,ast.Del)
  425. and inst.opname.startswith("DELETE_")
  426. and mangled_name(node) == inst.argval
  427. ):
  428. # Deleting the variable is valid and no exception cleanup, if the name is correct
  429. return False
  430. return any(
  431. isinstance(n, ast.ExceptHandler) and n.name and mangled_name(n) == inst.argval
  432. for n in parents(node)
  433. )
  434. def verify(self, node: EnhancedAST, instruction: dis.Instruction) -> None:
  435. """
  436. checks if this node could gererate this instruction
  437. """
  438. op_name = instruction.opname
  439. extra_filter: Callable[[EnhancedAST], bool] = lambda e: True
  440. ctx: Type = type(None)
  441. def inst_match(opnames: Union[str, Sequence[str]], **kwargs: Any) -> bool:
  442. """
  443. match instruction
  444. Parameters:
  445. opnames: (str|Seq[str]): inst.opname has to be equal to or in `opname`
  446. **kwargs: every arg has to match inst.arg
  447. Returns:
  448. True if all conditions match the instruction
  449. """
  450. if isinstance(opnames, str):
  451. opnames = [opnames]
  452. return instruction.opname in opnames and kwargs == {
  453. k: getattr(instruction, k) for k in kwargs
  454. }
  455. def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
  456. """
  457. match the ast-node
  458. Parameters:
  459. node_type: type of the node
  460. **kwargs: every `arg` has to be equal `node.arg`
  461. or `node.arg` has to be an instance of `arg` if it is a type.
  462. """
  463. return isinstance(node, node_type) and all(
  464. isinstance(getattr(node, k), v)
  465. if isinstance(v, type)
  466. else getattr(node, k) == v
  467. for k, v in kwargs.items()
  468. )
  469. if op_name == "CACHE":
  470. return
  471. if inst_match("CALL") and node_match((ast.With, ast.AsyncWith)):
  472. # call to context.__exit__
  473. return
  474. if inst_match(("CALL", "LOAD_FAST","LOAD_FAST_BORROW")) and node_match(
  475. (ast.ListComp, ast.GeneratorExp, ast.SetComp, ast.DictComp)
  476. ):
  477. # call to the generator function
  478. return
  479. if (
  480. sys.version_info >= (3, 12)
  481. and inst_match(("LOAD_FAST_AND_CLEAR", "STORE_FAST"))
  482. and node_match((ast.ListComp, ast.SetComp, ast.DictComp))
  483. ):
  484. return
  485. if inst_match(("CALL", "CALL_FUNCTION_EX")) and node_match(
  486. (ast.ClassDef, ast.Call)
  487. ):
  488. return
  489. if inst_match(("COMPARE_OP", "IS_OP", "CONTAINS_OP")) and node_match(
  490. ast.Compare
  491. ):
  492. return
  493. if inst_match("LOAD_NAME", argval="__annotations__") and node_match(
  494. ast.AnnAssign
  495. ):
  496. return
  497. if (
  498. (
  499. inst_match("LOAD_METHOD", argval="join")
  500. or inst_match("LOAD_ATTR", argval="join") # 3.12
  501. or inst_match(("CALL", "BUILD_STRING"))
  502. )
  503. and node_match(ast.BinOp, left=ast.Constant, op=ast.Mod)
  504. and isinstance(cast(ast.Constant, cast(ast.BinOp, node).left).value, str)
  505. ):
  506. # "..."%(...) uses "".join
  507. return
  508. if inst_match("STORE_SUBSCR") and node_match(ast.AnnAssign):
  509. # data: int
  510. return
  511. if inst_match(("DELETE_NAME", "DELETE_FAST")) and node_match(
  512. ast.Name, id=instruction.argval, ctx=ast.Del
  513. ):
  514. return
  515. if inst_match("BUILD_STRING") and (
  516. node_match(ast.JoinedStr) or node_match(ast.BinOp, op=ast.Mod)
  517. ):
  518. return
  519. if inst_match(("BEFORE_WITH","WITH_EXCEPT_START")) and node_match(ast.With):
  520. return
  521. if inst_match(("STORE_NAME", "STORE_GLOBAL"), argval="__doc__") and node_match(
  522. ast.Constant
  523. ):
  524. # store docstrings
  525. return
  526. if (
  527. inst_match(("STORE_NAME", "STORE_FAST", "STORE_GLOBAL", "STORE_DEREF"))
  528. and node_match(ast.ExceptHandler)
  529. and instruction.argval == mangled_name(node)
  530. ):
  531. # store exception in variable
  532. return
  533. if (
  534. inst_match(("STORE_NAME", "STORE_FAST", "STORE_DEREF", "STORE_GLOBAL"))
  535. and node_match((ast.Import, ast.ImportFrom))
  536. and any(mangled_name(cast(EnhancedAST, alias)) == instruction.argval for alias in cast(ast.Import, node).names)
  537. ):
  538. # store imported module in variable
  539. return
  540. if (
  541. inst_match(("STORE_FAST", "STORE_DEREF", "STORE_NAME", "STORE_GLOBAL"))
  542. and (
  543. node_match((ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef))
  544. or node_match(
  545. ast.Name,
  546. ctx=ast.Store,
  547. )
  548. )
  549. and instruction.argval == mangled_name(node)
  550. ):
  551. return
  552. if False:
  553. # TODO: match expressions are not supported for now
  554. if inst_match(("STORE_FAST", "STORE_NAME")) and node_match(
  555. ast.MatchAs, name=instruction.argval
  556. ):
  557. return
  558. if inst_match("COMPARE_OP", argval="==") and node_match(ast.MatchSequence):
  559. return
  560. if inst_match("COMPARE_OP", argval="==") and node_match(ast.MatchValue):
  561. return
  562. if inst_match("BINARY_OP"):
  563. arg=instruction.argrepr.removesuffix("=")
  564. if arg!="[]" and node_match( ast.AugAssign, op=op_type_map[arg]):
  565. # a+=5
  566. return
  567. if node_match(ast.Attribute, ctx=ast.Del) and inst_match(
  568. "DELETE_ATTR", argval=mangled_name(node)
  569. ):
  570. return
  571. if inst_match(
  572. (
  573. "JUMP_IF_TRUE_OR_POP",
  574. "JUMP_IF_FALSE_OR_POP",
  575. "POP_JUMP_IF_TRUE",
  576. "POP_JUMP_IF_FALSE",
  577. )
  578. ) and node_match(ast.BoolOp):
  579. # and/or short circuit
  580. return
  581. if inst_match("DELETE_SUBSCR") and node_match(ast.Subscript, ctx=ast.Del):
  582. return
  583. if (
  584. node_match(ast.Name, ctx=ast.Load)
  585. or (
  586. node_match(ast.Name, ctx=ast.Store)
  587. and isinstance(node.parent, ast.AugAssign)
  588. )
  589. ) and inst_match(
  590. (
  591. "LOAD_NAME",
  592. "LOAD_FAST",
  593. "LOAD_FAST_CHECK",
  594. "LOAD_FAST_BORROW",
  595. "LOAD_GLOBAL",
  596. "LOAD_DEREF",
  597. "LOAD_FROM_DICT_OR_DEREF",
  598. "LOAD_FAST_BORROW_LOAD_FAST_BORROW",
  599. ),
  600. ) and (
  601. mangled_name(node) in instruction.argval if isinstance(instruction.argval,tuple)
  602. else instruction.argval == mangled_name(node)
  603. ):
  604. return
  605. if node_match(ast.Name, ctx=ast.Del) and inst_match(
  606. ("DELETE_NAME", "DELETE_GLOBAL", "DELETE_DEREF"), argval=mangled_name(node)
  607. ):
  608. return
  609. if node_match(ast.Constant) and inst_match(
  610. ("LOAD_CONST","LOAD_SMALL_INT"), argval=cast(ast.Constant, node).value
  611. ):
  612. return
  613. if node_match(
  614. (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp, ast.For)
  615. ) and inst_match(("GET_ITER", "FOR_ITER")):
  616. return
  617. if sys.version_info >= (3, 12):
  618. if node_match(ast.UnaryOp, op=ast.UAdd) and inst_match(
  619. "CALL_INTRINSIC_1", argrepr="INTRINSIC_UNARY_POSITIVE"
  620. ):
  621. return
  622. if node_match(ast.Subscript) and inst_match("BINARY_SLICE"):
  623. return
  624. if node_match(ast.ImportFrom) and inst_match(
  625. "CALL_INTRINSIC_1", argrepr="INTRINSIC_IMPORT_STAR"
  626. ):
  627. return
  628. if (
  629. node_match(ast.Yield) or isinstance(node.parent, ast.GeneratorExp)
  630. ) and inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_ASYNC_GEN_WRAP"):
  631. return
  632. if node_match(ast.Name) and inst_match("LOAD_DEREF",argval="__classdict__"):
  633. return
  634. if node_match(ast.TypeVar) and (
  635. inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVAR")
  636. or inst_match(
  637. "CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_BOUND"
  638. )
  639. or inst_match(
  640. "CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_CONSTRAINTS"
  641. )
  642. or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=mangled_name(node))
  643. ):
  644. return
  645. if node_match(ast.TypeVarTuple) and (
  646. inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVARTUPLE")
  647. or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name)
  648. ):
  649. return
  650. if node_match(ast.ParamSpec) and (
  651. inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_PARAMSPEC")
  652. or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name)):
  653. return
  654. if node_match(ast.TypeAlias):
  655. if(
  656. inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEALIAS")
  657. or inst_match(
  658. ("STORE_NAME", "STORE_FAST", "STORE_DEREF","STORE_GLOBAL"), argrepr=node.name.id
  659. )
  660. or inst_match("CALL")
  661. ):
  662. return
  663. if node_match(ast.ClassDef) and node.type_params:
  664. if inst_match(
  665. ("STORE_DEREF", "LOAD_DEREF", "LOAD_FROM_DICT_OR_DEREF"),
  666. argrepr=".type_params",
  667. ):
  668. return
  669. if inst_match(("STORE_FAST", "LOAD_FAST"), argrepr=".generic_base"):
  670. return
  671. if inst_match(
  672. "CALL_INTRINSIC_1", argrepr="INTRINSIC_SUBSCRIPT_GENERIC"
  673. ):
  674. return
  675. if inst_match("LOAD_DEREF",argval="__classdict__"):
  676. return
  677. if node_match((ast.FunctionDef,ast.AsyncFunctionDef)) and node.type_params:
  678. if inst_match("CALL"):
  679. return
  680. if inst_match(
  681. "CALL_INTRINSIC_2", argrepr="INTRINSIC_SET_FUNCTION_TYPE_PARAMS"
  682. ):
  683. return
  684. if inst_match("LOAD_FAST",argval=".defaults"):
  685. return
  686. if inst_match("LOAD_FAST",argval=".kwdefaults"):
  687. return
  688. if sys.version_info >= (3, 14):
  689. if inst_match("LOAD_FAST_BORROW_LOAD_FAST_BORROW",argval=(".defaults",".kwdefaults")):
  690. return
  691. if inst_match("STORE_NAME", argval="__classdictcell__"):
  692. # this is a general thing
  693. return
  694. # f-strings
  695. if node_match(ast.JoinedStr) and (
  696. inst_match("LOAD_ATTR", argval="join")
  697. or inst_match(("LIST_APPEND", "CALL"))
  698. ):
  699. return
  700. if node_match(ast.FormattedValue) and inst_match("FORMAT_VALUE"):
  701. return
  702. if sys.version_info >= (3, 13):
  703. if inst_match("NOP"):
  704. return
  705. if inst_match("TO_BOOL") and node_match(ast.BoolOp):
  706. return
  707. if inst_match("CALL_KW") and node_match((ast.Call, ast.ClassDef)):
  708. return
  709. if inst_match("LOAD_FAST", argval=".type_params"):
  710. return
  711. if inst_match("LOAD_FAST", argval="__classdict__"):
  712. return
  713. if inst_match(("LOAD_FAST","LOAD_FAST_BORROW")) and node_match(
  714. (
  715. ast.FunctionDef,
  716. ast.ClassDef,
  717. ast.TypeAlias,
  718. ast.TypeVar,
  719. ast.Lambda,
  720. ast.AsyncFunctionDef,
  721. )
  722. ):
  723. # These are loads for closure variables.
  724. # It is difficult to check that this is actually closure variable, see:
  725. # https://github.com/alexmojaki/executing/pull/80#discussion_r1716027317
  726. return
  727. if (
  728. inst_match("LOAD_FAST")
  729. and node_match(ast.TypeAlias)
  730. and node.name.id == instruction.argval
  731. ):
  732. return
  733. if inst_match("STORE_NAME",argval="__static_attributes__"):
  734. # the node is the first node in the body
  735. return
  736. if inst_match(("LOAD_FAST","LOAD_FAST_BORROW")) and isinstance(node.parent,ast.TypeVar):
  737. return
  738. if inst_match("CALL_INTRINSIC_2",argrepr="INTRINSIC_SET_TYPEPARAM_DEFAULT") and node_match((ast.TypeVar,ast.ParamSpec,ast.TypeVarTuple)):
  739. return
  740. if sys.version_info >= (3, 14):
  741. if inst_match("BINARY_OP",argrepr="[]") and node_match(ast.Subscript):
  742. return
  743. if inst_match("LOAD_FAST_BORROW", argval="__classdict__"):
  744. return
  745. if inst_match(("STORE_NAME","LOAD_NAME"), argval="__conditional_annotations__"):
  746. return
  747. if inst_match("LOAD_FAST_BORROW_LOAD_FAST_BORROW") and node_match(ast.Name) and node.id in instruction.argval:
  748. return
  749. # old verifier
  750. typ: Type = type(None)
  751. op_type: Type = type(None)
  752. if op_name.startswith(("BINARY_SUBSCR", "SLICE+")):
  753. typ = ast.Subscript
  754. ctx = ast.Load
  755. elif op_name.startswith("BINARY_"):
  756. typ = ast.BinOp
  757. op_type = op_type_map[instruction.argrepr]
  758. extra_filter = lambda e: isinstance(cast(ast.BinOp, e).op, op_type)
  759. elif op_name.startswith("UNARY_"):
  760. typ = ast.UnaryOp
  761. op_type = dict(
  762. UNARY_POSITIVE=ast.UAdd,
  763. UNARY_NEGATIVE=ast.USub,
  764. UNARY_NOT=ast.Not,
  765. UNARY_INVERT=ast.Invert,
  766. )[op_name]
  767. extra_filter = lambda e: isinstance(cast(ast.UnaryOp, e).op, op_type)
  768. elif op_name in ("LOAD_ATTR", "LOAD_METHOD", "LOOKUP_METHOD","LOAD_SUPER_ATTR"):
  769. typ = ast.Attribute
  770. ctx = ast.Load
  771. extra_filter = lambda e: mangled_name(e) == instruction.argval
  772. elif op_name in (
  773. "LOAD_NAME",
  774. "LOAD_GLOBAL",
  775. "LOAD_FAST",
  776. "LOAD_DEREF",
  777. "LOAD_CLASSDEREF",
  778. ):
  779. typ = ast.Name
  780. ctx = ast.Load
  781. extra_filter = lambda e: cast(ast.Name, e).id == instruction.argval
  782. elif op_name in ("COMPARE_OP", "IS_OP", "CONTAINS_OP"):
  783. typ = ast.Compare
  784. extra_filter = lambda e: len(cast(ast.Compare, e).ops) == 1
  785. elif op_name.startswith(("STORE_SLICE", "STORE_SUBSCR")):
  786. ctx = ast.Store
  787. typ = ast.Subscript
  788. elif op_name.startswith("STORE_ATTR"):
  789. ctx = ast.Store
  790. typ = ast.Attribute
  791. extra_filter = lambda e: mangled_name(e) == instruction.argval
  792. node_ctx = getattr(node, "ctx", None)
  793. ctx_match = (
  794. ctx is not type(None)
  795. or not hasattr(node, "ctx")
  796. or isinstance(node_ctx, ctx)
  797. )
  798. # check for old verifier
  799. if isinstance(node, typ) and ctx_match and extra_filter(node):
  800. return
  801. # generate error
  802. title = "ast.%s is not created from %s" % (
  803. type(node).__name__,
  804. instruction.opname,
  805. )
  806. raise VerifierFailure(title, node, instruction)
  807. def instruction(self, index: int) -> Optional[dis.Instruction]:
  808. return self.bc_dict.get(index,None)
  809. def instruction_before(
  810. self, instruction: dis.Instruction
  811. ) -> Optional[dis.Instruction]:
  812. return self.bc_dict.get(instruction.offset - 2, None)
  813. def opname(self, index: int) -> str:
  814. i=self.instruction(index)
  815. if i is None:
  816. return "CACHE"
  817. return i.opname
  818. extra_node_types=()
  819. if sys.version_info >= (3,12):
  820. extra_node_types = (ast.type_param,)
  821. def find_node(
  822. self,
  823. index: int,
  824. match_positions: Sequence[str] = (
  825. "lineno",
  826. "end_lineno",
  827. "col_offset",
  828. "end_col_offset",
  829. ),
  830. typ: tuple[Type, ...] = (
  831. ast.expr,
  832. ast.stmt,
  833. ast.excepthandler,
  834. ast.pattern,
  835. *extra_node_types,
  836. ),
  837. ) -> EnhancedAST:
  838. instruction = self.instruction(index)
  839. assert instruction is not None
  840. position = instruction.positions
  841. assert position is not None and position.lineno is not None
  842. return only(
  843. cast(EnhancedAST, node)
  844. for node in self.source._nodes_by_line[position.lineno]
  845. if isinstance(node, typ)
  846. if not isinstance(node, ast.Expr)
  847. # matchvalue.value has the same positions as matchvalue themself, so we exclude ast.MatchValue
  848. if not isinstance(node, ast.MatchValue)
  849. if all(
  850. getattr(position, attr) == getattr(node, attr)
  851. for attr in match_positions
  852. )
  853. )