_check_docs_utils.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941
  1. # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
  2. # For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE
  3. # Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt
  4. """Utility methods for docstring checking."""
  5. from __future__ import annotations
  6. import itertools
  7. import re
  8. from collections.abc import Iterable
  9. import astroid
  10. from astroid import nodes
  11. from astroid.util import UninferableBase
  12. from pylint.checkers import utils
  13. def space_indentation(s: str) -> int:
  14. """The number of leading spaces in a string.
  15. :param str s: input string
  16. :rtype: int
  17. :return: number of leading spaces
  18. """
  19. return len(s) - len(s.lstrip(" "))
  20. def get_setters_property_name(node: nodes.FunctionDef) -> str | None:
  21. """Get the name of the property that the given node is a setter for.
  22. :param node: The node to get the property name for.
  23. :type node: str
  24. :rtype: str or None
  25. :returns: The name of the property that the node is a setter for,
  26. or None if one could not be found.
  27. """
  28. decorators = node.decorators.nodes if node.decorators else []
  29. for decorator in decorators:
  30. match decorator:
  31. case nodes.Attribute(attrname="setter", expr=nodes.Name(name=name)):
  32. return name # type: ignore[no-any-return]
  33. return None
  34. def get_setters_property(node: nodes.FunctionDef) -> nodes.FunctionDef | None:
  35. """Get the property node for the given setter node.
  36. :param node: The node to get the property for.
  37. :type node: nodes.FunctionDef
  38. :rtype: nodes.FunctionDef or None
  39. :returns: The node relating to the property of the given setter node,
  40. or None if one could not be found.
  41. """
  42. property_ = None
  43. property_name = get_setters_property_name(node)
  44. class_node = utils.node_frame_class(node)
  45. if property_name and class_node:
  46. class_attrs: list[nodes.FunctionDef] = class_node.getattr(node.name)
  47. for attr in class_attrs:
  48. if utils.decorated_with_property(attr):
  49. property_ = attr
  50. break
  51. return property_
  52. def returns_something(return_node: nodes.Return) -> bool:
  53. """Check if a return node returns a value other than None.
  54. :param return_node: The return node to check.
  55. :type return_node: nodes.Return
  56. :rtype: bool
  57. :return: True if the return node returns a value other than None,
  58. False otherwise.
  59. """
  60. returns = return_node.value
  61. return not (
  62. returns is None or (isinstance(returns, nodes.Const) and returns.value is None)
  63. )
  64. def _get_raise_target(node: nodes.NodeNG) -> nodes.NodeNG | UninferableBase | None:
  65. match node.exc:
  66. case nodes.Call(func=nodes.Name() | nodes.Attribute() as func):
  67. return utils.safe_infer(func)
  68. return None
  69. def _split_multiple_exc_types(target: str) -> list[str]:
  70. delimiters = r"(\s*,(?:\s*or\s)?\s*|\s+or\s+)"
  71. return re.split(delimiters, target)
  72. def possible_exc_types(node: nodes.NodeNG) -> set[nodes.ClassDef]:
  73. """Gets all the possible raised exception types for the given raise node.
  74. .. note::
  75. Caught exception types are ignored.
  76. :param node: The raise node to find exception types for.
  77. :returns: A list of exception types possibly raised by :param:`node`.
  78. """
  79. exceptions = []
  80. if isinstance(node.exc, nodes.Name):
  81. inferred = utils.safe_infer(node.exc)
  82. if inferred:
  83. exceptions = [inferred]
  84. elif node.exc is None:
  85. handler = node.parent
  86. while handler and not isinstance(handler, nodes.ExceptHandler):
  87. handler = handler.parent
  88. if handler and handler.type:
  89. try:
  90. for exception in astroid.unpack_infer(handler.type):
  91. if not isinstance(exception, UninferableBase):
  92. exceptions.append(exception)
  93. except astroid.InferenceError:
  94. pass
  95. else:
  96. match target := _get_raise_target(node):
  97. case nodes.ClassDef():
  98. exceptions = [target]
  99. case nodes.FunctionDef():
  100. for ret in target.nodes_of_class(nodes.Return):
  101. if ret.value is None:
  102. continue
  103. if ret.frame() != target:
  104. # return from inner function - ignore it
  105. continue
  106. val = utils.safe_infer(ret.value)
  107. if val and utils.inherit_from_std_ex(val):
  108. match val:
  109. case nodes.ClassDef():
  110. exceptions.append(val)
  111. case astroid.Instance():
  112. exceptions.append(val.getattr("__class__")[0])
  113. try:
  114. return {
  115. exc
  116. for exc in exceptions
  117. if not utils.node_ignores_exception(node, exc.name)
  118. }
  119. except astroid.InferenceError:
  120. return set()
  121. def _is_ellipsis(node: nodes.NodeNG) -> bool:
  122. return isinstance(node, nodes.Const) and node.value == Ellipsis
  123. def _merge_annotations(
  124. annotations: Iterable[nodes.NodeNG], comment_annotations: Iterable[nodes.NodeNG]
  125. ) -> Iterable[nodes.NodeNG | None]:
  126. for ann, comment_ann in itertools.zip_longest(annotations, comment_annotations):
  127. if ann and not _is_ellipsis(ann):
  128. yield ann
  129. elif comment_ann and not _is_ellipsis(comment_ann):
  130. yield comment_ann
  131. else:
  132. yield None
  133. def _annotations_list(args_node: nodes.Arguments) -> list[nodes.NodeNG]:
  134. """Get a merged list of annotations.
  135. The annotations can come from:
  136. * Real type annotations.
  137. * A type comment on the function.
  138. * A type common on the individual argument.
  139. :param args_node: The node to get the annotations for.
  140. :returns: The annotations.
  141. """
  142. plain_annotations = args_node.annotations or ()
  143. func_comment_annotations = args_node.parent.type_comment_args or ()
  144. comment_annotations = args_node.type_comment_posonlyargs
  145. comment_annotations += args_node.type_comment_args or []
  146. comment_annotations += args_node.type_comment_kwonlyargs
  147. return list(
  148. _merge_annotations(
  149. plain_annotations,
  150. _merge_annotations(func_comment_annotations, comment_annotations),
  151. )
  152. )
  153. def args_with_annotation(args_node: nodes.Arguments) -> set[str]:
  154. result = set()
  155. annotations = _annotations_list(args_node)
  156. annotation_offset = 0
  157. if args_node.posonlyargs:
  158. posonlyargs_annotations = args_node.posonlyargs_annotations
  159. if not any(args_node.posonlyargs_annotations):
  160. num_args = len(args_node.posonlyargs)
  161. posonlyargs_annotations = annotations[
  162. annotation_offset : annotation_offset + num_args
  163. ]
  164. annotation_offset += num_args
  165. for arg, annotation in zip(args_node.posonlyargs, posonlyargs_annotations):
  166. if annotation:
  167. result.add(arg.name)
  168. if args_node.args:
  169. num_args = len(args_node.args)
  170. for arg, annotation in zip(
  171. args_node.args,
  172. annotations[annotation_offset : annotation_offset + num_args],
  173. ):
  174. if annotation:
  175. result.add(arg.name)
  176. annotation_offset += num_args
  177. if args_node.vararg:
  178. if args_node.varargannotation:
  179. result.add(args_node.vararg)
  180. elif len(annotations) > annotation_offset and annotations[annotation_offset]:
  181. result.add(args_node.vararg)
  182. annotation_offset += 1
  183. if args_node.kwonlyargs:
  184. kwonlyargs_annotations = args_node.kwonlyargs_annotations
  185. if not any(args_node.kwonlyargs_annotations):
  186. num_args = len(args_node.kwonlyargs)
  187. kwonlyargs_annotations = annotations[
  188. annotation_offset : annotation_offset + num_args
  189. ]
  190. annotation_offset += num_args
  191. for arg, annotation in zip(args_node.kwonlyargs, kwonlyargs_annotations):
  192. if annotation:
  193. result.add(arg.name)
  194. if args_node.kwarg:
  195. if args_node.kwargannotation:
  196. result.add(args_node.kwarg)
  197. elif len(annotations) > annotation_offset and annotations[annotation_offset]:
  198. result.add(args_node.kwarg)
  199. annotation_offset += 1
  200. return result
  201. def docstringify(
  202. docstring: nodes.Const | None, default_type: str = "default"
  203. ) -> Docstring:
  204. best_match = (0, DOCSTRING_TYPES.get(default_type, Docstring)(docstring))
  205. for docstring_type in (
  206. SphinxDocstring,
  207. EpytextDocstring,
  208. GoogleDocstring,
  209. NumpyDocstring,
  210. ):
  211. instance = docstring_type(docstring)
  212. matching_sections = instance.matching_sections()
  213. if matching_sections > best_match[0]:
  214. best_match = (matching_sections, instance)
  215. return best_match[1]
  216. class Docstring:
  217. re_for_parameters_see = re.compile(
  218. r"""
  219. For\s+the\s+(other)?\s*parameters\s*,\s+see
  220. """,
  221. re.X | re.S,
  222. )
  223. supports_yields: bool = False
  224. """True if the docstring supports a "yield" section.
  225. False if the docstring uses the returns section to document generators.
  226. """
  227. # These methods are designed to be overridden
  228. def __init__(self, doc: nodes.Const | None) -> None:
  229. docstring: str = doc.value if doc else ""
  230. self.doc = docstring.expandtabs()
  231. def __repr__(self) -> str:
  232. return f"<{self.__class__.__name__}:'''{self.doc}'''>"
  233. def matching_sections(self) -> int:
  234. """Returns the number of matching docstring sections."""
  235. return 0
  236. def exceptions(self) -> set[str]:
  237. return set()
  238. def has_params(self) -> bool:
  239. return False
  240. def has_returns(self) -> bool:
  241. return False
  242. def has_rtype(self) -> bool:
  243. return False
  244. def has_property_returns(self) -> bool:
  245. return False
  246. def has_property_type(self) -> bool:
  247. return False
  248. def has_yields(self) -> bool:
  249. return False
  250. def has_yields_type(self) -> bool:
  251. return False
  252. def match_param_docs(self) -> tuple[set[str], set[str]]:
  253. return set(), set()
  254. def params_documented_elsewhere(self) -> bool:
  255. return self.re_for_parameters_see.search(self.doc) is not None
  256. class SphinxDocstring(Docstring):
  257. re_type = r"""
  258. [~!.]? # Optional link style prefix
  259. \w(?:\w|\.[^\.])* # Valid python name
  260. """
  261. re_simple_container_type = rf"""
  262. {re_type} # a container type
  263. [\(\[] [^\n\s]+ [\)\]] # with the contents of the container
  264. """
  265. re_multiple_simple_type = rf"""
  266. (?:{re_simple_container_type}|{re_type})
  267. (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{re_simple_container_type}|{re_type}))*
  268. """
  269. re_xref = rf"""
  270. (?::\w+:)? # optional tag
  271. `{re_type}` # what to reference
  272. """
  273. re_param_raw = rf"""
  274. : # initial colon
  275. (?: # Sphinx keywords
  276. param|parameter|
  277. arg|argument|
  278. key|keyword
  279. )
  280. \s+ # whitespace
  281. (?: # optional type declaration
  282. ({re_type}|{re_simple_container_type})
  283. \s+
  284. )?
  285. ((\\\*{{0,2}}\w+)|(\w+)) # Parameter name with potential asterisks
  286. \s* # whitespace
  287. : # final colon
  288. """
  289. re_param_in_docstring = re.compile(re_param_raw, re.X | re.S)
  290. re_type_raw = rf"""
  291. :type # Sphinx keyword
  292. \s+ # whitespace
  293. ({re_multiple_simple_type}) # Parameter name
  294. \s* # whitespace
  295. : # final colon
  296. """
  297. re_type_in_docstring = re.compile(re_type_raw, re.X | re.S)
  298. re_property_type_raw = rf"""
  299. :type: # Sphinx keyword
  300. \s+ # whitespace
  301. {re_multiple_simple_type} # type declaration
  302. """
  303. re_property_type_in_docstring = re.compile(re_property_type_raw, re.X | re.S)
  304. re_raise_raw = rf"""
  305. : # initial colon
  306. (?: # Sphinx keyword
  307. raises?|
  308. except|exception
  309. )
  310. \s+ # whitespace
  311. ({re_multiple_simple_type}) # exception type
  312. \s* # whitespace
  313. : # final colon
  314. """
  315. re_raise_in_docstring = re.compile(re_raise_raw, re.X | re.S)
  316. re_rtype_in_docstring = re.compile(r":rtype:")
  317. re_returns_in_docstring = re.compile(r":returns?:")
  318. supports_yields = False
  319. def matching_sections(self) -> int:
  320. """Returns the number of matching docstring sections."""
  321. return sum(
  322. bool(i)
  323. for i in (
  324. self.re_param_in_docstring.search(self.doc),
  325. self.re_raise_in_docstring.search(self.doc),
  326. self.re_rtype_in_docstring.search(self.doc),
  327. self.re_returns_in_docstring.search(self.doc),
  328. self.re_property_type_in_docstring.search(self.doc),
  329. )
  330. )
  331. def exceptions(self) -> set[str]:
  332. types: set[str] = set()
  333. for match in re.finditer(self.re_raise_in_docstring, self.doc):
  334. raise_type = match.group(1)
  335. types.update(_split_multiple_exc_types(raise_type))
  336. return types
  337. def has_params(self) -> bool:
  338. if not self.doc:
  339. return False
  340. return self.re_param_in_docstring.search(self.doc) is not None
  341. def has_returns(self) -> bool:
  342. if not self.doc:
  343. return False
  344. return bool(self.re_returns_in_docstring.search(self.doc))
  345. def has_rtype(self) -> bool:
  346. if not self.doc:
  347. return False
  348. return bool(self.re_rtype_in_docstring.search(self.doc))
  349. def has_property_returns(self) -> bool:
  350. if not self.doc:
  351. return False
  352. # The summary line is the return doc,
  353. # so the first line must not be a known directive.
  354. return not self.doc.lstrip().startswith(":")
  355. def has_property_type(self) -> bool:
  356. if not self.doc:
  357. return False
  358. return bool(self.re_property_type_in_docstring.search(self.doc))
  359. def match_param_docs(self) -> tuple[set[str], set[str]]:
  360. params_with_doc = set()
  361. params_with_type = set()
  362. for match in re.finditer(self.re_param_in_docstring, self.doc):
  363. name = match.group(2)
  364. # Remove escape characters necessary for asterisks
  365. name = name.replace("\\", "")
  366. params_with_doc.add(name)
  367. param_type = match.group(1)
  368. if param_type is not None:
  369. params_with_type.add(name)
  370. params_with_type.update(re.findall(self.re_type_in_docstring, self.doc))
  371. return params_with_doc, params_with_type
  372. class EpytextDocstring(SphinxDocstring):
  373. """Epytext is similar to Sphinx.
  374. See the docs:
  375. http://epydoc.sourceforge.net/epytext.html
  376. http://epydoc.sourceforge.net/fields.html#fields
  377. It's used in PyCharm:
  378. https://www.jetbrains.com/help/pycharm/2016.1/creating-documentation-comments.html#d848203e314
  379. https://www.jetbrains.com/help/pycharm/2016.1/using-docstrings-to-specify-types.html
  380. """
  381. re_param_in_docstring = re.compile(
  382. SphinxDocstring.re_param_raw.replace(":", "@", 1), re.X | re.S
  383. )
  384. re_type_in_docstring = re.compile(
  385. SphinxDocstring.re_type_raw.replace(":", "@", 1), re.X | re.S
  386. )
  387. re_property_type_in_docstring = re.compile(
  388. SphinxDocstring.re_property_type_raw.replace(":", "@", 1), re.X | re.S
  389. )
  390. re_raise_in_docstring = re.compile(
  391. SphinxDocstring.re_raise_raw.replace(":", "@", 1), re.X | re.S
  392. )
  393. re_rtype_in_docstring = re.compile(
  394. r"""
  395. @ # initial "at" symbol
  396. (?: # Epytext keyword
  397. rtype|returntype
  398. )
  399. : # final colon
  400. """,
  401. re.X | re.S,
  402. )
  403. re_returns_in_docstring = re.compile(r"@returns?:")
  404. def has_property_returns(self) -> bool:
  405. if not self.doc:
  406. return False
  407. # If this is a property docstring, the summary is the return doc.
  408. if self.has_property_type():
  409. # The summary line is the return doc,
  410. # so the first line must not be a known directive.
  411. return not self.doc.lstrip().startswith("@")
  412. return False
  413. class GoogleDocstring(Docstring):
  414. re_type = SphinxDocstring.re_type
  415. re_xref = SphinxDocstring.re_xref
  416. re_container_type = rf"""
  417. (?:{re_type}|{re_xref}) # a container type
  418. [\(\[] [^\n]+ [\)\]] # with the contents of the container
  419. """
  420. re_multiple_type = rf"""
  421. (?:{re_container_type}|{re_type}|{re_xref})
  422. (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{re_container_type}|{re_type}|{re_xref}))*
  423. """
  424. _re_section_template = r"""
  425. ^([ ]*) {0} \s*: \s*$ # Google parameter header
  426. ( .* ) # section
  427. """
  428. re_param_section = re.compile(
  429. _re_section_template.format(r"(?:Args|Arguments|Parameters)"),
  430. re.X | re.S | re.M,
  431. )
  432. re_keyword_param_section = re.compile(
  433. _re_section_template.format(r"Keyword\s(?:Args|Arguments|Parameters)"),
  434. re.X | re.S | re.M,
  435. )
  436. re_param_line = re.compile(
  437. rf"""
  438. \s* ((?:\\?\*{{0,2}})?[\w\\]+) # identifier potentially with asterisks or escaped `\`
  439. \s* ( [(]
  440. {re_multiple_type}
  441. (?:,\s+optional)?
  442. [)] )? \s* : # optional type declaration
  443. \s* (.*) # beginning of optional description
  444. """,
  445. re.X | re.S | re.M,
  446. )
  447. re_raise_section = re.compile(
  448. _re_section_template.format(r"Raises"), re.X | re.S | re.M
  449. )
  450. re_raise_line = re.compile(
  451. rf"""
  452. \s* ({re_multiple_type}) \s* : # identifier
  453. \s* (.*) # beginning of optional description
  454. """,
  455. re.X | re.S | re.M,
  456. )
  457. re_returns_section = re.compile(
  458. _re_section_template.format(r"Returns?"), re.X | re.S | re.M
  459. )
  460. re_returns_line = re.compile(
  461. rf"""
  462. \s* ({re_multiple_type}:)? # identifier
  463. \s* (.*) # beginning of description
  464. """,
  465. re.X | re.S | re.M,
  466. )
  467. re_property_returns_line = re.compile(
  468. rf"""
  469. ^{re_multiple_type}: # identifier
  470. \s* (.*) # Summary line / description
  471. """,
  472. re.X | re.S | re.M,
  473. )
  474. re_yields_section = re.compile(
  475. _re_section_template.format(r"Yields?"), re.X | re.S | re.M
  476. )
  477. re_yields_line = re_returns_line
  478. supports_yields = True
  479. def matching_sections(self) -> int:
  480. """Returns the number of matching docstring sections."""
  481. return sum(
  482. bool(i)
  483. for i in (
  484. self.re_param_section.search(self.doc),
  485. self.re_raise_section.search(self.doc),
  486. self.re_returns_section.search(self.doc),
  487. self.re_yields_section.search(self.doc),
  488. self.re_property_returns_line.search(self._first_line()),
  489. )
  490. )
  491. def has_params(self) -> bool:
  492. if not self.doc:
  493. return False
  494. return self.re_param_section.search(self.doc) is not None
  495. def has_returns(self) -> bool:
  496. if not self.doc:
  497. return False
  498. entries = self._parse_section(self.re_returns_section)
  499. for entry in entries:
  500. match = self.re_returns_line.match(entry)
  501. if not match:
  502. continue
  503. return_desc = match.group(2)
  504. if return_desc:
  505. return True
  506. return False
  507. def has_rtype(self) -> bool:
  508. if not self.doc:
  509. return False
  510. entries = self._parse_section(self.re_returns_section)
  511. for entry in entries:
  512. match = self.re_returns_line.match(entry)
  513. if not match:
  514. continue
  515. return_type = match.group(1)
  516. if return_type:
  517. return True
  518. return False
  519. def has_property_returns(self) -> bool:
  520. # The summary line is the return doc,
  521. # so the first line must not be a known directive.
  522. first_line = self._first_line()
  523. return not bool(
  524. self.re_param_section.search(first_line)
  525. or self.re_raise_section.search(first_line)
  526. or self.re_returns_section.search(first_line)
  527. or self.re_yields_section.search(first_line)
  528. )
  529. def has_property_type(self) -> bool:
  530. if not self.doc:
  531. return False
  532. return bool(self.re_property_returns_line.match(self._first_line()))
  533. def has_yields(self) -> bool:
  534. if not self.doc:
  535. return False
  536. entries = self._parse_section(self.re_yields_section)
  537. for entry in entries:
  538. match = self.re_yields_line.match(entry)
  539. if not match:
  540. continue
  541. yield_desc = match.group(2)
  542. if yield_desc:
  543. return True
  544. return False
  545. def has_yields_type(self) -> bool:
  546. if not self.doc:
  547. return False
  548. entries = self._parse_section(self.re_yields_section)
  549. for entry in entries:
  550. match = self.re_yields_line.match(entry)
  551. if not match:
  552. continue
  553. yield_type = match.group(1)
  554. if yield_type:
  555. return True
  556. return False
  557. def exceptions(self) -> set[str]:
  558. types: set[str] = set()
  559. entries = self._parse_section(self.re_raise_section)
  560. for entry in entries:
  561. match = self.re_raise_line.match(entry)
  562. if not match:
  563. continue
  564. exc_type = match.group(1)
  565. exc_desc = match.group(2)
  566. if exc_desc:
  567. types.update(_split_multiple_exc_types(exc_type))
  568. return types
  569. def match_param_docs(self) -> tuple[set[str], set[str]]:
  570. params_with_doc: set[str] = set()
  571. params_with_type: set[str] = set()
  572. entries = self._parse_section(self.re_param_section)
  573. entries.extend(self._parse_section(self.re_keyword_param_section))
  574. for entry in entries:
  575. match = self.re_param_line.match(entry)
  576. if not match:
  577. continue
  578. param_name = match.group(1)
  579. # Remove escape characters necessary for asterisks
  580. param_name = param_name.replace("\\", "")
  581. param_type = match.group(2)
  582. param_desc = match.group(3)
  583. if param_type:
  584. params_with_type.add(param_name)
  585. if param_desc:
  586. params_with_doc.add(param_name)
  587. return params_with_doc, params_with_type
  588. def _first_line(self) -> str:
  589. return self.doc.lstrip().split("\n", 1)[0]
  590. @staticmethod
  591. def min_section_indent(section_match: re.Match[str]) -> int:
  592. return len(section_match.group(1)) + 1
  593. @staticmethod
  594. def _is_section_header(_: str) -> bool:
  595. # Google parsing does not need to detect section headers,
  596. # because it works off of indentation level only
  597. return False
  598. def _parse_section(self, section_re: re.Pattern[str]) -> list[str]:
  599. section_match = section_re.search(self.doc)
  600. if section_match is None:
  601. return []
  602. min_indentation = self.min_section_indent(section_match)
  603. entries: list[str] = []
  604. entry: list[str] = []
  605. is_first = True
  606. for line in section_match.group(2).splitlines():
  607. if not line.strip():
  608. continue
  609. indentation = space_indentation(line)
  610. if indentation < min_indentation:
  611. break
  612. # The first line after the header defines the minimum
  613. # indentation.
  614. if is_first:
  615. min_indentation = indentation
  616. is_first = False
  617. if indentation == min_indentation:
  618. if self._is_section_header(line):
  619. break
  620. # Lines with minimum indentation must contain the beginning
  621. # of a new parameter documentation.
  622. if entry:
  623. entries.append("\n".join(entry))
  624. entry = []
  625. entry.append(line)
  626. if entry:
  627. entries.append("\n".join(entry))
  628. return entries
  629. class NumpyDocstring(GoogleDocstring):
  630. _re_section_template = r"""
  631. ^([ ]*) {0} \s*?$ # Numpy parameters header
  632. \s* [-=]+ \s*?$ # underline
  633. ( .* ) # section
  634. """
  635. re_param_section = re.compile(
  636. _re_section_template.format(r"(?:Args|Arguments|Parameters)"),
  637. re.X | re.S | re.M,
  638. )
  639. re_default_value = r"""((['"]\w+\s*['"])|(\d+)|(True)|(False)|(None))"""
  640. re_param_line = re.compile(
  641. rf"""
  642. \s* (?P<param_name>\*{{0,2}}\w+)(\s?(:|\n)) # identifier with potential asterisks
  643. \s*
  644. (?P<param_type>
  645. (
  646. ({GoogleDocstring.re_multiple_type}) # default type declaration
  647. (,\s+optional)? # optional 'optional' indication
  648. )?
  649. (
  650. {{({re_default_value},?\s*)+}} # set of default values
  651. )?
  652. (?:$|\n)
  653. )?
  654. (
  655. \s* (?P<param_desc>.*) # optional description
  656. )?
  657. """,
  658. re.X | re.S,
  659. )
  660. re_raise_section = re.compile(
  661. _re_section_template.format(r"Raises"), re.X | re.S | re.M
  662. )
  663. re_raise_line = re.compile(
  664. rf"""
  665. \s* ({GoogleDocstring.re_type})$ # type declaration
  666. \s* (.*) # optional description
  667. """,
  668. re.X | re.S | re.M,
  669. )
  670. re_returns_section = re.compile(
  671. _re_section_template.format(r"Returns?"), re.X | re.S | re.M
  672. )
  673. re_returns_line = re.compile(
  674. rf"""
  675. \s* (?:\w+\s+:\s+)? # optional name
  676. ({GoogleDocstring.re_multiple_type})$ # type declaration
  677. \s* (.*) # optional description
  678. """,
  679. re.X | re.S | re.M,
  680. )
  681. re_yields_section = re.compile(
  682. _re_section_template.format(r"Yields?"), re.X | re.S | re.M
  683. )
  684. re_yields_line = re_returns_line
  685. supports_yields = True
  686. def match_param_docs(self) -> tuple[set[str], set[str]]:
  687. """Matches parameter documentation section to parameter documentation rules."""
  688. params_with_doc = set()
  689. params_with_type = set()
  690. entries = self._parse_section(self.re_param_section)
  691. entries.extend(self._parse_section(self.re_keyword_param_section))
  692. for entry in entries:
  693. match = self.re_param_line.match(entry)
  694. if not match:
  695. continue
  696. # check if parameter has description only
  697. re_only_desc = re.match(r"\s*(\*{0,2}\w+)\s*:?\n\s*\w*$", entry)
  698. if re_only_desc:
  699. param_name = match.group("param_name")
  700. param_desc = match.group("param_type")
  701. param_type = None
  702. else:
  703. param_name = match.group("param_name")
  704. param_type = match.group("param_type")
  705. param_desc = match.group("param_desc")
  706. # The re_param_line pattern needs to match multi-line which removes the ability
  707. # to match a single line description like 'arg : a number type.'
  708. # We are not trying to determine whether 'a number type' is correct typing
  709. # but we do accept it as typing as it is in the place where typing
  710. # should be
  711. if param_type is None and re.match(r"\s*(\*{0,2}\w+)\s*:.+$", entry):
  712. param_type = param_desc
  713. # If the description is "" but we have a type description
  714. # we consider the description to be the type
  715. if not param_desc and param_type:
  716. param_desc = param_type
  717. if param_type:
  718. params_with_type.add(param_name)
  719. if param_desc:
  720. params_with_doc.add(param_name)
  721. return params_with_doc, params_with_type
  722. @staticmethod
  723. def min_section_indent(section_match: re.Match[str]) -> int:
  724. return len(section_match.group(1))
  725. @staticmethod
  726. def _is_section_header(line: str) -> bool:
  727. return bool(re.match(r"\s*-+$", line))
  728. DOCSTRING_TYPES = {
  729. "sphinx": SphinxDocstring,
  730. "epytext": EpytextDocstring,
  731. "google": GoogleDocstring,
  732. "numpy": NumpyDocstring,
  733. "default": Docstring,
  734. }
  735. """A map of the name of the docstring type to its class.
  736. :type: dict(str, type)
  737. """