_ast_gen.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. # -----------------------------------------------------------------
  2. # _ast_gen.py
  3. #
  4. # Generates the AST Node classes from a specification given in
  5. # a configuration file. This module can also be run as a script to
  6. # regenerate c_ast.py from _c_ast.cfg (from the repo root or the
  7. # pycparser/ directory). Use 'make check' to reformat the generated
  8. # file after running this script.
  9. #
  10. # The design of this module was inspired by astgen.py from the
  11. # Python 2.5 code-base.
  12. #
  13. # Eli Bendersky [https://eli.thegreenplace.net/]
  14. # License: BSD
  15. # -----------------------------------------------------------------
  16. from string import Template
  17. import os
  18. from typing import IO
  19. class ASTCodeGenerator:
  20. def __init__(self, cfg_filename="_c_ast.cfg"):
  21. """Initialize the code generator from a configuration
  22. file.
  23. """
  24. self.cfg_filename = cfg_filename
  25. self.node_cfg = [
  26. NodeCfg(name, contents)
  27. for (name, contents) in self.parse_cfgfile(cfg_filename)
  28. ]
  29. def generate(self, file: IO[str]) -> None:
  30. """Generates the code into file, an open file buffer."""
  31. src = Template(_PROLOGUE_COMMENT).substitute(cfg_filename=self.cfg_filename)
  32. src += _PROLOGUE_CODE
  33. for node_cfg in self.node_cfg:
  34. src += node_cfg.generate_source() + "\n\n"
  35. file.write(src)
  36. def parse_cfgfile(self, filename):
  37. """Parse the configuration file and yield pairs of
  38. (name, contents) for each node.
  39. """
  40. with open(filename, "r") as f:
  41. for line in f:
  42. line = line.strip()
  43. if not line or line.startswith("#"):
  44. continue
  45. colon_i = line.find(":")
  46. lbracket_i = line.find("[")
  47. rbracket_i = line.find("]")
  48. if colon_i < 1 or lbracket_i <= colon_i or rbracket_i <= lbracket_i:
  49. raise RuntimeError(f"Invalid line in {filename}:\n{line}\n")
  50. name = line[:colon_i]
  51. val = line[lbracket_i + 1 : rbracket_i]
  52. vallist = [v.strip() for v in val.split(",")] if val else []
  53. yield name, vallist
  54. class NodeCfg:
  55. """Node configuration.
  56. name: node name
  57. contents: a list of contents - attributes and child nodes
  58. See comment at the top of the configuration file for details.
  59. """
  60. def __init__(self, name, contents):
  61. self.name = name
  62. self.all_entries = []
  63. self.attr = []
  64. self.child = []
  65. self.seq_child = []
  66. for entry in contents:
  67. clean_entry = entry.rstrip("*")
  68. self.all_entries.append(clean_entry)
  69. if entry.endswith("**"):
  70. self.seq_child.append(clean_entry)
  71. elif entry.endswith("*"):
  72. self.child.append(clean_entry)
  73. else:
  74. self.attr.append(entry)
  75. def generate_source(self):
  76. src = self._gen_init()
  77. src += "\n" + self._gen_children()
  78. src += "\n" + self._gen_iter()
  79. src += "\n" + self._gen_attr_names()
  80. return src
  81. def _gen_init(self):
  82. src = f"class {self.name}(Node):\n"
  83. if self.all_entries:
  84. args = ", ".join(self.all_entries)
  85. slots = ", ".join(f"'{e}'" for e in self.all_entries)
  86. slots += ", 'coord', '__weakref__'"
  87. arglist = f"(self, {args}, coord=None)"
  88. else:
  89. slots = "'coord', '__weakref__'"
  90. arglist = "(self, coord=None)"
  91. src += f" __slots__ = ({slots})\n"
  92. src += f" def __init__{arglist}:\n"
  93. for name in self.all_entries + ["coord"]:
  94. src += f" self.{name} = {name}\n"
  95. return src
  96. def _gen_children(self):
  97. src = " def children(self):\n"
  98. if self.all_entries:
  99. src += " nodelist = []\n"
  100. for child in self.child:
  101. src += f" if self.{child} is not None:\n"
  102. src += f' nodelist.append(("{child}", self.{child}))\n'
  103. for seq_child in self.seq_child:
  104. src += f" for i, child in enumerate(self.{seq_child} or []):\n"
  105. src += f' nodelist.append((f"{seq_child}[{{i}}]", child))\n'
  106. src += " return tuple(nodelist)\n"
  107. else:
  108. src += " return ()\n"
  109. return src
  110. def _gen_iter(self):
  111. src = " def __iter__(self):\n"
  112. if self.all_entries:
  113. for child in self.child:
  114. src += f" if self.{child} is not None:\n"
  115. src += f" yield self.{child}\n"
  116. for seq_child in self.seq_child:
  117. src += f" for child in (self.{seq_child} or []):\n"
  118. src += " yield child\n"
  119. if not (self.child or self.seq_child):
  120. # Empty generator
  121. src += " return\n" + " yield\n"
  122. else:
  123. # Empty generator
  124. src += " return\n" + " yield\n"
  125. return src
  126. def _gen_attr_names(self):
  127. src = " attr_names = (" + "".join(f"{nm!r}, " for nm in self.attr) + ")"
  128. return src
  129. _PROLOGUE_COMMENT = r"""#-----------------------------------------------------------------
  130. # ** ATTENTION **
  131. # This code was automatically generated from _c_ast.cfg
  132. #
  133. # Do not modify it directly. Modify the configuration file and
  134. # run the generator again.
  135. # ** ** *** ** **
  136. #
  137. # pycparser: c_ast.py
  138. #
  139. # AST Node classes.
  140. #
  141. # Eli Bendersky [https://eli.thegreenplace.net/]
  142. # License: BSD
  143. #-----------------------------------------------------------------
  144. """
  145. _PROLOGUE_CODE = r'''
  146. import sys
  147. from typing import Any, ClassVar, IO, Optional
  148. def _repr(obj):
  149. """
  150. Get the representation of an object, with dedicated pprint-like format for lists.
  151. """
  152. if isinstance(obj, list):
  153. return '[' + (',\n '.join((_repr(e).replace('\n', '\n ') for e in obj))) + '\n]'
  154. else:
  155. return repr(obj)
  156. class Node:
  157. __slots__ = ()
  158. """ Abstract base class for AST nodes.
  159. """
  160. attr_names: ClassVar[tuple[str, ...]] = ()
  161. coord: Optional[Any]
  162. def __repr__(self):
  163. """ Generates a python representation of the current node
  164. """
  165. result = self.__class__.__name__ + '('
  166. indent = ''
  167. separator = ''
  168. for name in self.__slots__[:-2]:
  169. result += separator
  170. result += indent
  171. result += name + '=' + (_repr(getattr(self, name)).replace('\n', '\n ' + (' ' * (len(name) + len(self.__class__.__name__)))))
  172. separator = ','
  173. indent = '\n ' + (' ' * len(self.__class__.__name__))
  174. result += indent + ')'
  175. return result
  176. def children(self):
  177. """ A sequence of all children that are Nodes
  178. """
  179. pass
  180. def show(
  181. self,
  182. buf: IO[str] = sys.stdout,
  183. offset: int = 0,
  184. attrnames: bool = False,
  185. showemptyattrs: bool = True,
  186. nodenames: bool = False,
  187. showcoord: bool = False,
  188. _my_node_name: Optional[str] = None,
  189. ):
  190. """ Pretty print the Node and all its attributes and
  191. children (recursively) to a buffer.
  192. buf:
  193. Open IO buffer into which the Node is printed.
  194. offset:
  195. Initial offset (amount of leading spaces)
  196. attrnames:
  197. True if you want to see the attribute names in
  198. name=value pairs. False to only see the values.
  199. showemptyattrs:
  200. False if you want to suppress printing empty attributes.
  201. nodenames:
  202. True if you want to see the actual node names
  203. within their parents.
  204. showcoord:
  205. Do you want the coordinates of each Node to be
  206. displayed.
  207. """
  208. lead = ' ' * offset
  209. if nodenames and _my_node_name is not None:
  210. buf.write(lead + self.__class__.__name__+ ' <' + _my_node_name + '>: ')
  211. else:
  212. buf.write(lead + self.__class__.__name__+ ': ')
  213. if self.attr_names:
  214. def is_empty(v):
  215. v is None or (hasattr(v, '__len__') and len(v) == 0)
  216. nvlist = [(n, getattr(self,n)) for n in self.attr_names \
  217. if showemptyattrs or not is_empty(getattr(self,n))]
  218. if attrnames:
  219. attrstr = ', '.join(f'{name}={value}' for name, value in nvlist)
  220. else:
  221. attrstr = ', '.join(f'{value}' for _, value in nvlist)
  222. buf.write(attrstr)
  223. if showcoord:
  224. buf.write(f' (at {self.coord})')
  225. buf.write('\n')
  226. for (child_name, child) in self.children():
  227. child.show(
  228. buf,
  229. offset=offset + 2,
  230. attrnames=attrnames,
  231. showemptyattrs=showemptyattrs,
  232. nodenames=nodenames,
  233. showcoord=showcoord,
  234. _my_node_name=child_name)
  235. class NodeVisitor:
  236. """ A base NodeVisitor class for visiting c_ast nodes.
  237. Subclass it and define your own visit_XXX methods, where
  238. XXX is the class name you want to visit with these
  239. methods.
  240. For example:
  241. class ConstantVisitor(NodeVisitor):
  242. def __init__(self):
  243. self.values = []
  244. def visit_Constant(self, node):
  245. self.values.append(node.value)
  246. Creates a list of values of all the constant nodes
  247. encountered below the given node. To use it:
  248. cv = ConstantVisitor()
  249. cv.visit(node)
  250. Notes:
  251. * generic_visit() will be called for AST nodes for which
  252. no visit_XXX method was defined.
  253. * The children of nodes for which a visit_XXX was
  254. defined will not be visited - if you need this, call
  255. generic_visit() on the node.
  256. You can use:
  257. NodeVisitor.generic_visit(self, node)
  258. * Modeled after Python's own AST visiting facilities
  259. (the ast module of Python 3.0)
  260. """
  261. _method_cache = None
  262. def visit(self, node: Node):
  263. """ Visit a node.
  264. """
  265. if self._method_cache is None:
  266. self._method_cache = {}
  267. visitor = self._method_cache.get(node.__class__.__name__, None)
  268. if visitor is None:
  269. method = 'visit_' + node.__class__.__name__
  270. visitor = getattr(self, method, self.generic_visit)
  271. self._method_cache[node.__class__.__name__] = visitor
  272. return visitor(node)
  273. def generic_visit(self, node: Node):
  274. """ Called if no explicit visitor function exists for a
  275. node. Implements preorder visiting of the node.
  276. """
  277. for _, c in node.children():
  278. self.visit(c)
  279. '''
  280. if __name__ == "__main__":
  281. base_dir = os.path.dirname(os.path.abspath(__file__))
  282. cfg_path = os.path.join(base_dir, "_c_ast.cfg")
  283. out_path = os.path.join(base_dir, "c_ast.py")
  284. ast_gen = ASTCodeGenerator(cfg_path)
  285. with open(out_path, "w") as out:
  286. ast_gen.generate(out)