imports.py 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279
  1. # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
  2. # For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE
  3. # Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt
  4. """Imports checkers for Python code."""
  5. from __future__ import annotations
  6. import collections
  7. import copy
  8. import os
  9. import sys
  10. from collections import defaultdict
  11. from collections.abc import ItemsView, Sequence
  12. from functools import cached_property
  13. from typing import TYPE_CHECKING, Any
  14. import astroid
  15. import astroid.modutils
  16. import isort
  17. from astroid import nodes
  18. from astroid.nodes._base_nodes import ImportNode
  19. from pylint.checkers import BaseChecker, DeprecatedMixin
  20. from pylint.checkers.utils import (
  21. get_import_name,
  22. in_type_checking_block,
  23. is_from_fallback_block,
  24. is_module_ignored,
  25. is_sys_guard,
  26. node_ignores_exception,
  27. )
  28. from pylint.constants import MAX_NUMBER_OF_IMPORT_SHOWN
  29. from pylint.exceptions import EmptyReportError
  30. from pylint.graph import DotBackend, get_cycles
  31. from pylint.interfaces import HIGH
  32. from pylint.reporters.ureports.nodes import Paragraph, Section, VerbatimText
  33. from pylint.typing import MessageDefinitionTuple
  34. from pylint.utils.linterstats import LinterStats
  35. if TYPE_CHECKING:
  36. from pylint.lint import PyLinter
  37. # The dictionary with Any should actually be a _ImportTree again
  38. # but mypy doesn't support recursive types yet
  39. _ImportTree = dict[str, list[dict[str, Any]] | list[str]]
  40. DEPRECATED_MODULES = {
  41. (0, 0, 0): {"tkinter.tix", "fpectl"},
  42. (3, 3, 0): {"xml.etree.cElementTree"},
  43. (3, 4, 0): {"imp"},
  44. (3, 5, 0): {"formatter"},
  45. (3, 6, 0): {"asynchat", "asyncore", "smtpd"},
  46. (3, 7, 0): {"macpath"},
  47. (3, 9, 0): {"lib2to3", "parser", "symbol", "binhex"},
  48. (3, 10, 0): {"distutils", "typing.io", "typing.re"},
  49. (3, 11, 0): {
  50. "aifc",
  51. "audioop",
  52. "cgi",
  53. "cgitb",
  54. "chunk",
  55. "crypt",
  56. "imghdr",
  57. "msilib",
  58. "mailcap",
  59. "nis",
  60. "nntplib",
  61. "ossaudiodev",
  62. "pipes",
  63. "sndhdr",
  64. "spwd",
  65. "sunau",
  66. "sre_compile",
  67. "sre_constants",
  68. "sre_parse",
  69. "telnetlib",
  70. "uu",
  71. "xdrlib",
  72. },
  73. }
  74. def _get_first_import(
  75. node: ImportNode,
  76. context: nodes.LocalsDictNodeNG,
  77. name: str,
  78. base: str | None,
  79. level: int | None,
  80. alias: str | None,
  81. ) -> tuple[nodes.Import | nodes.ImportFrom | None, str | None]:
  82. """Return the node where [base.]<name> is imported or None if not found."""
  83. fullname = f"{base}.{name}" if base else name
  84. first = None
  85. found = False
  86. msg = "reimported"
  87. for first in context.body:
  88. if first is node:
  89. continue
  90. if first.scope() is node.scope() and first.fromlineno > node.fromlineno:
  91. continue
  92. if isinstance(first, nodes.Import):
  93. if any(fullname == iname[0] for iname in first.names):
  94. found = True
  95. break
  96. for imported_name, imported_alias in first.names:
  97. if not imported_alias and imported_name == alias:
  98. found = True
  99. msg = "shadowed-import"
  100. break
  101. if found:
  102. break
  103. elif isinstance(first, nodes.ImportFrom):
  104. if level == first.level:
  105. for imported_name, imported_alias in first.names:
  106. if fullname == f"{first.modname}.{imported_name}":
  107. found = True
  108. break
  109. if (
  110. name != "*"
  111. and name == imported_name
  112. and not (alias or imported_alias)
  113. ):
  114. found = True
  115. break
  116. if not imported_alias and imported_name == alias:
  117. found = True
  118. msg = "shadowed-import"
  119. break
  120. if found:
  121. break
  122. if found and not astroid.are_exclusive(first, node):
  123. return first, msg
  124. return None, None
  125. def _ignore_import_failure(
  126. node: ImportNode,
  127. modname: str,
  128. ignored_modules: Sequence[str],
  129. ) -> bool:
  130. if is_module_ignored(modname, ignored_modules):
  131. return True
  132. # Ignore import failure if part of guarded import block
  133. # I.e. `sys.version_info` or `typing.TYPE_CHECKING`
  134. if in_type_checking_block(node):
  135. return True
  136. if isinstance(node.parent, nodes.If) and is_sys_guard(node.parent):
  137. return True
  138. return node_ignores_exception(node, ImportError)
  139. # utilities to represents import dependencies as tree and dot graph ###########
  140. def _make_tree_defs(mod_files_list: ItemsView[str, set[str]]) -> _ImportTree:
  141. """Get a list of 2-uple (module, list_of_files_which_import_this_module),
  142. it will return a dictionary to represent this as a tree.
  143. """
  144. tree_defs: _ImportTree = {}
  145. for mod, files in mod_files_list:
  146. node: list[_ImportTree | list[str]] = [tree_defs, []]
  147. for prefix in mod.split("."):
  148. assert isinstance(node[0], dict)
  149. node = node[0].setdefault(prefix, ({}, [])) # type: ignore[arg-type,assignment]
  150. assert isinstance(node[1], list)
  151. node[1].extend(files)
  152. return tree_defs
  153. def _repr_tree_defs(data: _ImportTree, indent_str: str | None = None) -> str:
  154. """Return a string which represents imports as a tree."""
  155. lines = []
  156. nodes_items = data.items()
  157. for i, (mod, (sub, files)) in enumerate(sorted(nodes_items, key=lambda x: x[0])):
  158. files_list = "" if not files else f"({','.join(sorted(files))})"
  159. if indent_str is None:
  160. lines.append(f"{mod} {files_list}")
  161. sub_indent_str = " "
  162. else:
  163. lines.append(rf"{indent_str}\-{mod} {files_list}")
  164. if i == len(nodes_items) - 1:
  165. sub_indent_str = f"{indent_str} "
  166. else:
  167. sub_indent_str = f"{indent_str}| "
  168. if sub and isinstance(sub, dict):
  169. lines.append(_repr_tree_defs(sub, sub_indent_str))
  170. return "\n".join(lines)
  171. def _dependencies_graph(filename: str, dep_info: dict[str, set[str]]) -> str:
  172. """Write dependencies as a dot (graphviz) file."""
  173. done = {}
  174. printer = DotBackend(os.path.splitext(os.path.basename(filename))[0], rankdir="LR")
  175. printer.emit('URL="." node[shape="box"]')
  176. for modname, dependencies in sorted(dep_info.items()):
  177. sorted_dependencies = sorted(dependencies)
  178. done[modname] = 1
  179. printer.emit_node(modname)
  180. for depmodname in sorted_dependencies:
  181. if depmodname not in done:
  182. done[depmodname] = 1
  183. printer.emit_node(depmodname)
  184. for depmodname, dependencies in sorted(dep_info.items()):
  185. for modname in sorted(dependencies):
  186. printer.emit_edge(modname, depmodname)
  187. return printer.generate(filename)
  188. def _make_graph(
  189. filename: str, dep_info: dict[str, set[str]], sect: Section, gtype: str
  190. ) -> None:
  191. """Generate a dependencies graph and add some information about it in the
  192. report's section.
  193. """
  194. outputfile = _dependencies_graph(filename, dep_info)
  195. sect.append(Paragraph((f"{gtype}imports graph has been written to {outputfile}",)))
  196. # the import checker itself ###################################################
  197. MSGS: dict[str, MessageDefinitionTuple] = {
  198. "E0401": (
  199. "Unable to import %s",
  200. "import-error",
  201. "Used when pylint has been unable to import a module.",
  202. {"old_names": [("F0401", "old-import-error")]},
  203. ),
  204. "E0402": (
  205. "Attempted relative import beyond top-level package",
  206. "relative-beyond-top-level",
  207. "Used when a relative import tries to access too many levels "
  208. "in the current package.",
  209. ),
  210. "R0401": (
  211. "Cyclic import (%s)",
  212. "cyclic-import",
  213. "Used when a cyclic import between two or more modules is detected.",
  214. ),
  215. "R0402": (
  216. "Use 'from %s import %s' instead",
  217. "consider-using-from-import",
  218. "Emitted when a submodule of a package is imported and "
  219. "aliased with the same name, "
  220. "e.g., instead of ``import concurrent.futures as futures`` use "
  221. "``from concurrent import futures``.",
  222. ),
  223. "W0401": (
  224. "Wildcard import %s",
  225. "wildcard-import",
  226. "Used when `from module import *` is detected.",
  227. ),
  228. "W0404": (
  229. "Reimport %r (imported line %s)",
  230. "reimported",
  231. "Used when a module is imported more than once.",
  232. ),
  233. "W0406": (
  234. "Module import itself",
  235. "import-self",
  236. "Used when a module is importing itself.",
  237. ),
  238. "W0407": (
  239. "Prefer importing %r instead of %r",
  240. "preferred-module",
  241. "Used when a module imported has a preferred replacement module.",
  242. ),
  243. "W0410": (
  244. "__future__ import is not the first non docstring statement",
  245. "misplaced-future",
  246. "Python 2.5 and greater require __future__ import to be the "
  247. "first non docstring statement in the module.",
  248. ),
  249. "C0410": (
  250. "Multiple imports on one line (%s)",
  251. "multiple-imports",
  252. "Used when import statement importing multiple modules is detected.",
  253. ),
  254. "C0411": (
  255. "%s should be placed before %s",
  256. "wrong-import-order",
  257. "Used when PEP8 import order is not respected (standard imports "
  258. "first, then third-party libraries, then local imports).",
  259. ),
  260. "C0412": (
  261. "Imports from package %s are not grouped",
  262. "ungrouped-imports",
  263. "Used when imports are not grouped by packages.",
  264. ),
  265. "C0413": (
  266. 'Import "%s" should be placed at the top of the module',
  267. "wrong-import-position",
  268. "Used when code and imports are mixed.",
  269. ),
  270. "C0414": (
  271. "Import alias does not rename original package",
  272. "useless-import-alias",
  273. "Used when an import alias is same as original package, "
  274. "e.g., using import numpy as numpy instead of import numpy as np.",
  275. ),
  276. "C0415": (
  277. "Import outside toplevel (%s)",
  278. "import-outside-toplevel",
  279. "Used when an import statement is used anywhere other than the module "
  280. "toplevel. Move this import to the top of the file.",
  281. ),
  282. "W0416": (
  283. "Shadowed %r (imported line %s)",
  284. "shadowed-import",
  285. "Used when a module is aliased with a name that shadows another import.",
  286. ),
  287. }
  288. DEFAULT_STANDARD_LIBRARY = ()
  289. DEFAULT_KNOWN_THIRD_PARTY = ("enchant",)
  290. DEFAULT_PREFERRED_MODULES = ()
  291. class ImportsChecker(DeprecatedMixin, BaseChecker):
  292. """BaseChecker for import statements.
  293. Checks for
  294. * external modules dependencies
  295. * relative / wildcard imports
  296. * cyclic imports
  297. * uses of deprecated modules
  298. * uses of modules instead of preferred modules
  299. """
  300. name = "imports"
  301. msgs = {**DeprecatedMixin.DEPRECATED_MODULE_MESSAGE, **MSGS}
  302. default_deprecated_modules = ()
  303. options = (
  304. (
  305. "deprecated-modules",
  306. {
  307. "default": default_deprecated_modules,
  308. "type": "csv",
  309. "metavar": "<modules>",
  310. "help": "Deprecated modules which should not be used,"
  311. " separated by a comma.",
  312. },
  313. ),
  314. (
  315. "preferred-modules",
  316. {
  317. "default": DEFAULT_PREFERRED_MODULES,
  318. "type": "csv",
  319. "metavar": "<module:preferred-module>",
  320. "help": "Couples of modules and preferred modules,"
  321. " separated by a comma.",
  322. },
  323. ),
  324. (
  325. "import-graph",
  326. {
  327. "default": "",
  328. "type": "path",
  329. "metavar": "<file.gv>",
  330. "help": "Output a graph (.gv or any supported image format) of"
  331. " all (i.e. internal and external) dependencies to the given file"
  332. " (report RP0402 must not be disabled).",
  333. },
  334. ),
  335. (
  336. "ext-import-graph",
  337. {
  338. "default": "",
  339. "type": "path",
  340. "metavar": "<file.gv>",
  341. "help": "Output a graph (.gv or any supported image format)"
  342. " of external dependencies to the given file"
  343. " (report RP0402 must not be disabled).",
  344. },
  345. ),
  346. (
  347. "int-import-graph",
  348. {
  349. "default": "",
  350. "type": "path",
  351. "metavar": "<file.gv>",
  352. "help": "Output a graph (.gv or any supported image format)"
  353. " of internal dependencies to the given file"
  354. " (report RP0402 must not be disabled).",
  355. },
  356. ),
  357. (
  358. "known-standard-library",
  359. {
  360. "default": DEFAULT_STANDARD_LIBRARY,
  361. "type": "csv",
  362. "metavar": "<modules>",
  363. "help": "Force import order to recognize a module as part of "
  364. "the standard compatibility libraries.",
  365. },
  366. ),
  367. (
  368. "known-third-party",
  369. {
  370. "default": DEFAULT_KNOWN_THIRD_PARTY,
  371. "type": "csv",
  372. "metavar": "<modules>",
  373. "help": "Force import order to recognize a module as part of "
  374. "a third party library.",
  375. },
  376. ),
  377. (
  378. "allow-any-import-level",
  379. {
  380. "default": (),
  381. "type": "csv",
  382. "metavar": "<modules>",
  383. "help": (
  384. "List of modules that can be imported at any level, not just "
  385. "the top level one."
  386. ),
  387. },
  388. ),
  389. (
  390. "allow-wildcard-with-all",
  391. {
  392. "default": False,
  393. "type": "yn",
  394. "metavar": "<y or n>",
  395. "help": "Allow wildcard imports from modules that define __all__.",
  396. },
  397. ),
  398. (
  399. "allow-reexport-from-package",
  400. {
  401. "default": False,
  402. "type": "yn",
  403. "metavar": "<y or n>",
  404. "help": "Allow explicit reexports by alias from a package __init__.",
  405. },
  406. ),
  407. )
  408. def __init__(self, linter: PyLinter) -> None:
  409. BaseChecker.__init__(self, linter)
  410. self.import_graph: defaultdict[str, set[str]] = defaultdict(set)
  411. self._imports_stack: list[tuple[ImportNode, str]] = []
  412. self._first_non_import_node = None
  413. self._module_pkg: dict[Any, Any] = (
  414. {}
  415. ) # mapping of modules to the pkg they belong in
  416. self._allow_any_import_level: set[Any] = set()
  417. self.reports = (
  418. ("RP0401", "External dependencies", self._report_external_dependencies),
  419. ("RP0402", "Modules dependencies graph", self._report_dependencies_graph),
  420. )
  421. self._excluded_edges: defaultdict[str, set[str]] = defaultdict(set)
  422. def open(self) -> None:
  423. """Called before visiting project (i.e set of modules)."""
  424. self.linter.stats.dependencies = {}
  425. self.linter.stats = self.linter.stats
  426. self.import_graph = defaultdict(set)
  427. self._module_pkg = {} # mapping of modules to the pkg they belong in
  428. self._current_module_package = False
  429. self._ignored_modules: Sequence[str] = self.linter.config.ignored_modules
  430. # Build a mapping {'module': 'preferred-module'}
  431. self.preferred_modules = dict(
  432. module.split(":")
  433. for module in self.linter.config.preferred_modules
  434. if ":" in module
  435. )
  436. self._allow_any_import_level = set(self.linter.config.allow_any_import_level)
  437. self._allow_reexport_package = self.linter.config.allow_reexport_from_package
  438. def _import_graph_without_ignored_edges(self) -> defaultdict[str, set[str]]:
  439. filtered_graph = copy.deepcopy(self.import_graph)
  440. for node in filtered_graph:
  441. filtered_graph[node].difference_update(self._excluded_edges[node])
  442. return filtered_graph
  443. def close(self) -> None:
  444. """Called before visiting project (i.e set of modules)."""
  445. if self.linter.is_message_enabled("cyclic-import"):
  446. graph = self._import_graph_without_ignored_edges()
  447. vertices = list(graph)
  448. for cycle in get_cycles(graph, vertices=vertices):
  449. self.add_message("cyclic-import", args=" -> ".join(cycle))
  450. def get_map_data(
  451. self,
  452. ) -> tuple[defaultdict[str, set[str]], defaultdict[str, set[str]]]:
  453. if self.linter.is_message_enabled("cyclic-import"):
  454. return (self.import_graph, self._excluded_edges)
  455. return (defaultdict(set), defaultdict(set))
  456. def reduce_map_data(
  457. self,
  458. linter: PyLinter,
  459. data: list[tuple[defaultdict[str, set[str]], defaultdict[str, set[str]]]],
  460. ) -> None:
  461. if self.linter.is_message_enabled("cyclic-import"):
  462. self.import_graph = defaultdict(set)
  463. self._excluded_edges = defaultdict(set)
  464. for to_update in data:
  465. graph, excluded_edges = to_update
  466. self.import_graph.update(graph)
  467. self._excluded_edges.update(excluded_edges)
  468. self.close()
  469. def deprecated_modules(self) -> set[str]:
  470. """Callback returning the deprecated modules."""
  471. # First get the modules the user indicated
  472. all_deprecated_modules = set(self.linter.config.deprecated_modules)
  473. # Now get the hard-coded ones from the stdlib
  474. for since_vers, mod_set in DEPRECATED_MODULES.items():
  475. if since_vers <= sys.version_info:
  476. all_deprecated_modules = all_deprecated_modules.union(mod_set)
  477. return all_deprecated_modules
  478. def visit_module(self, node: nodes.Module) -> None:
  479. """Store if current module is a package, i.e. an __init__ file."""
  480. self._current_module_package = node.package
  481. def visit_import(self, node: nodes.Import) -> None:
  482. """Triggered when an import statement is seen."""
  483. self._check_reimport(node)
  484. self._check_import_as_rename(node)
  485. self._check_toplevel(node)
  486. names = [name for name, _ in node.names]
  487. if len(names) >= 2:
  488. self.add_message("multiple-imports", args=", ".join(names), node=node)
  489. for name in names:
  490. self.check_deprecated_module(node, name)
  491. self._check_preferred_module(node, name)
  492. imported_module = self._get_imported_module(node, name)
  493. if isinstance(node.parent, nodes.Module):
  494. # Allow imports nested
  495. self._check_position(node)
  496. if isinstance(node.scope(), nodes.Module):
  497. self._record_import(node, imported_module)
  498. if imported_module is None:
  499. continue
  500. self._add_imported_module(node, imported_module.name)
  501. def visit_importfrom(self, node: nodes.ImportFrom) -> None:
  502. """Triggered when a from statement is seen."""
  503. basename = node.modname
  504. imported_module = self._get_imported_module(node, basename)
  505. absolute_name = get_import_name(node, basename)
  506. self._check_import_as_rename(node)
  507. self._check_misplaced_future(node)
  508. self.check_deprecated_module(node, absolute_name)
  509. self._check_preferred_module(node, basename)
  510. self._check_wildcard_imports(node, imported_module)
  511. self._check_same_line_imports(node)
  512. self._check_reimport(node, basename=basename, level=node.level)
  513. self._check_toplevel(node)
  514. if isinstance(node.parent, nodes.Module):
  515. # Allow imports nested
  516. self._check_position(node)
  517. if isinstance(node.scope(), nodes.Module):
  518. self._record_import(node, imported_module)
  519. if imported_module is None:
  520. return
  521. for name, _ in node.names:
  522. if name != "*":
  523. self._add_imported_module(node, f"{imported_module.name}.{name}")
  524. else:
  525. self._add_imported_module(node, imported_module.name)
  526. def leave_module(self, node: nodes.Module) -> None:
  527. # Check imports are grouped by category (standard, 3rd party, local)
  528. std_imports, ext_imports, loc_imports = self._check_imports_order(node)
  529. # Check that imports are grouped by package within a given category
  530. met_import: set[str] = set() # set for 'import x' style
  531. met_from: set[str] = set() # set for 'from x import y' style
  532. current_package = None
  533. for import_node, import_name in std_imports + ext_imports + loc_imports:
  534. met = met_from if isinstance(import_node, nodes.ImportFrom) else met_import
  535. package, _, _ = import_name.partition(".")
  536. if (
  537. current_package
  538. and current_package != package
  539. and package in met
  540. and not in_type_checking_block(import_node)
  541. and not (
  542. isinstance(import_node.parent, nodes.If)
  543. and is_sys_guard(import_node.parent)
  544. )
  545. ):
  546. self.add_message("ungrouped-imports", node=import_node, args=package)
  547. current_package = package
  548. if not self.linter.is_message_enabled(
  549. "ungrouped-imports", import_node.fromlineno
  550. ):
  551. continue
  552. met.add(package)
  553. self._imports_stack = []
  554. self._first_non_import_node = None
  555. def compute_first_non_import_node(
  556. self,
  557. node: (
  558. nodes.If
  559. | nodes.Expr
  560. | nodes.Comprehension
  561. | nodes.IfExp
  562. | nodes.Assign
  563. | nodes.AssignAttr
  564. | nodes.Try
  565. ),
  566. ) -> None:
  567. # if the node does not contain an import instruction, and if it is the
  568. # first node of the module, keep a track of it (all the import positions
  569. # of the module will be compared to the position of this first
  570. # instruction)
  571. if self._first_non_import_node:
  572. return
  573. if not isinstance(node.parent, nodes.Module):
  574. return
  575. if isinstance(node, nodes.Try) and any(
  576. node.nodes_of_class((nodes.Import, nodes.ImportFrom))
  577. ):
  578. return
  579. if isinstance(node, nodes.Assign):
  580. # Add compatibility for module level dunder names
  581. # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
  582. valid_targets = [
  583. isinstance(target, nodes.AssignName)
  584. and target.name.startswith("__")
  585. and target.name.endswith("__")
  586. for target in node.targets
  587. ]
  588. if all(valid_targets):
  589. return
  590. self._first_non_import_node = node
  591. visit_try = visit_assignattr = visit_assign = visit_ifexp = visit_comprehension = (
  592. visit_expr
  593. ) = visit_if = compute_first_non_import_node
  594. def visit_functiondef(
  595. self, node: nodes.FunctionDef | nodes.While | nodes.For | nodes.ClassDef
  596. ) -> None:
  597. # If it is the first non import instruction of the module, record it.
  598. if self._first_non_import_node:
  599. return
  600. # Check if the node belongs to an `If` or a `Try` block. If they
  601. # contain imports, skip recording this node.
  602. if not isinstance(node.parent.scope(), nodes.Module):
  603. return
  604. root = node
  605. while not isinstance(root.parent, nodes.Module):
  606. root = root.parent
  607. if isinstance(root, (nodes.If, nodes.Try)):
  608. if any(root.nodes_of_class((nodes.Import, nodes.ImportFrom))):
  609. return
  610. self._first_non_import_node = node
  611. visit_classdef = visit_for = visit_while = visit_functiondef
  612. def _check_misplaced_future(self, node: nodes.ImportFrom) -> None:
  613. basename = node.modname
  614. if basename == "__future__":
  615. # check if this is the first non-docstring statement in the module
  616. prev = node.previous_sibling()
  617. if prev:
  618. # consecutive future statements are possible
  619. if not (
  620. isinstance(prev, nodes.ImportFrom) and prev.modname == "__future__"
  621. ):
  622. self.add_message("misplaced-future", node=node)
  623. def _check_same_line_imports(self, node: nodes.ImportFrom) -> None:
  624. # Detect duplicate imports on the same line.
  625. names = (name for name, _ in node.names)
  626. counter = collections.Counter(names)
  627. for name, count in counter.items():
  628. if count > 1:
  629. self.add_message("reimported", node=node, args=(name, node.fromlineno))
  630. def _check_position(self, node: ImportNode) -> None:
  631. """Check `node` import or importfrom node position is correct.
  632. Send a message if `node` comes before another instruction
  633. """
  634. # if a first non-import instruction has already been encountered,
  635. # it means the import comes after it and therefore is not well placed
  636. if self._first_non_import_node:
  637. if self.linter.is_message_enabled(
  638. "wrong-import-position", self._first_non_import_node.fromlineno
  639. ):
  640. self.add_message(
  641. "wrong-import-position", node=node, args=node.as_string()
  642. )
  643. else:
  644. self.linter.add_ignored_message(
  645. "wrong-import-position", node.fromlineno, node
  646. )
  647. def _record_import(
  648. self,
  649. node: ImportNode,
  650. importedmodnode: nodes.Module | None,
  651. ) -> None:
  652. """Record the package `node` imports from."""
  653. if isinstance(node, nodes.ImportFrom):
  654. importedname = node.modname
  655. else:
  656. importedname = importedmodnode.name if importedmodnode else None
  657. if not importedname:
  658. importedname = node.names[0][0].split(".")[0]
  659. if isinstance(node, nodes.ImportFrom) and (node.level or 0) >= 1:
  660. # We need the importedname with first point to detect local package
  661. # Example of node:
  662. # 'from .my_package1 import MyClass1'
  663. # the output should be '.my_package1' instead of 'my_package1'
  664. # Example of node:
  665. # 'from . import my_package2'
  666. # the output should be '.my_package2' instead of '{pyfile}'
  667. importedname = "." + importedname
  668. self._imports_stack.append((node, importedname))
  669. @staticmethod
  670. def _is_fallback_import(
  671. node: ImportNode, imports: list[tuple[ImportNode, str]]
  672. ) -> bool:
  673. imports = [import_node for (import_node, _) in imports]
  674. return any(astroid.are_exclusive(import_node, node) for import_node in imports)
  675. @property
  676. def _isort_config(self) -> isort.Config:
  677. """Get the config for use with isort.
  678. Only valid after CLI parsing finished, i.e. not in __init__
  679. """
  680. return isort.Config(
  681. # There is no typo here. EXTRA_standard_library is
  682. # what most users want. The option has been named
  683. # KNOWN_standard_library for ages in pylint, and we
  684. # don't want to break compatibility.
  685. extra_standard_library=self.linter.config.known_standard_library,
  686. known_third_party=self.linter.config.known_third_party,
  687. )
  688. def _check_imports_order(self, _module_node: nodes.Module) -> tuple[
  689. list[tuple[ImportNode, str]],
  690. list[tuple[ImportNode, str]],
  691. list[tuple[ImportNode, str]],
  692. ]:
  693. """Checks imports of module `node` are grouped by category.
  694. Imports must follow this order: standard, 3rd party, 1st party, local
  695. """
  696. std_imports: list[tuple[ImportNode, str]] = []
  697. third_party_imports: list[tuple[ImportNode, str]] = []
  698. first_party_imports: list[tuple[ImportNode, str]] = []
  699. # need of a list that holds third or first party ordered import
  700. external_imports: list[tuple[ImportNode, str]] = []
  701. local_imports: list[tuple[ImportNode, str]] = []
  702. third_party_not_ignored: list[tuple[ImportNode, str]] = []
  703. first_party_not_ignored: list[tuple[ImportNode, str]] = []
  704. local_not_ignored: list[tuple[ImportNode, str]] = []
  705. for node, modname in self._imports_stack:
  706. if modname.startswith("."):
  707. package = "." + modname.split(".")[1]
  708. else:
  709. package = modname.split(".")[0]
  710. nested = not isinstance(node.parent, nodes.Module)
  711. ignore_for_import_order = not self.linter.is_message_enabled(
  712. "wrong-import-order", node.fromlineno
  713. )
  714. import_category = isort.place_module(package, config=self._isort_config)
  715. node_and_package_import = (node, package)
  716. match import_category:
  717. case "FUTURE" | "STDLIB":
  718. std_imports.append(node_and_package_import)
  719. wrong_import = (
  720. third_party_not_ignored
  721. or first_party_not_ignored
  722. or local_not_ignored
  723. )
  724. if self._is_fallback_import(node, wrong_import):
  725. continue
  726. if wrong_import and not nested:
  727. self.add_message(
  728. "wrong-import-order",
  729. node=node,
  730. args=( ## TODO - this isn't right for multiple on the same line...
  731. f'standard import "{self._get_full_import_name((node, package))}"',
  732. self._get_out_of_order_string(
  733. third_party_not_ignored,
  734. first_party_not_ignored,
  735. local_not_ignored,
  736. ),
  737. ),
  738. )
  739. case "THIRDPARTY":
  740. third_party_imports.append(node_and_package_import)
  741. external_imports.append(node_and_package_import)
  742. if not nested:
  743. if not ignore_for_import_order:
  744. third_party_not_ignored.append(node_and_package_import)
  745. else:
  746. self.linter.add_ignored_message(
  747. "wrong-import-order", node.fromlineno, node
  748. )
  749. wrong_import = first_party_not_ignored or local_not_ignored
  750. if wrong_import and not nested:
  751. self.add_message(
  752. "wrong-import-order",
  753. node=node,
  754. args=(
  755. f'third party import "{self._get_full_import_name((node, package))}"',
  756. self._get_out_of_order_string(
  757. None, first_party_not_ignored, local_not_ignored
  758. ),
  759. ),
  760. )
  761. case "FIRSTPARTY":
  762. first_party_imports.append(node_and_package_import)
  763. external_imports.append(node_and_package_import)
  764. if not nested:
  765. if not ignore_for_import_order:
  766. first_party_not_ignored.append(node_and_package_import)
  767. else:
  768. self.linter.add_ignored_message(
  769. "wrong-import-order", node.fromlineno, node
  770. )
  771. wrong_import = local_not_ignored
  772. if wrong_import and not nested:
  773. self.add_message(
  774. "wrong-import-order",
  775. node=node,
  776. args=(
  777. f'first party import "{self._get_full_import_name((node, package))}"',
  778. self._get_out_of_order_string(
  779. None, None, local_not_ignored
  780. ),
  781. ),
  782. )
  783. case "LOCALFOLDER":
  784. local_imports.append((node, package))
  785. if not nested:
  786. if not ignore_for_import_order:
  787. local_not_ignored.append((node, package))
  788. else:
  789. self.linter.add_ignored_message(
  790. "wrong-import-order", node.fromlineno, node
  791. )
  792. return std_imports, external_imports, local_imports
  793. def _get_out_of_order_string(
  794. self,
  795. third_party_imports: list[tuple[ImportNode, str]] | None,
  796. first_party_imports: list[tuple[ImportNode, str]] | None,
  797. local_imports: list[tuple[ImportNode, str]] | None,
  798. ) -> str:
  799. # construct the string listing out of order imports used in the message
  800. # for wrong-import-order
  801. if third_party_imports:
  802. plural = "s" if len(third_party_imports) > 1 else ""
  803. if len(third_party_imports) > MAX_NUMBER_OF_IMPORT_SHOWN:
  804. imports_list = (
  805. ", ".join(
  806. [
  807. f'"{self._get_full_import_name(tpi)}"'
  808. for tpi in third_party_imports[
  809. : int(MAX_NUMBER_OF_IMPORT_SHOWN // 2)
  810. ]
  811. ]
  812. )
  813. + " (...) "
  814. + ", ".join(
  815. [
  816. f'"{self._get_full_import_name(tpi)}"'
  817. for tpi in third_party_imports[
  818. int(-MAX_NUMBER_OF_IMPORT_SHOWN // 2) :
  819. ]
  820. ]
  821. )
  822. )
  823. else:
  824. imports_list = ", ".join(
  825. [
  826. f'"{self._get_full_import_name(tpi)}"'
  827. for tpi in third_party_imports
  828. ]
  829. )
  830. third_party = f"third party import{plural} {imports_list}"
  831. else:
  832. third_party = ""
  833. if first_party_imports:
  834. plural = "s" if len(first_party_imports) > 1 else ""
  835. if len(first_party_imports) > MAX_NUMBER_OF_IMPORT_SHOWN:
  836. imports_list = (
  837. ", ".join(
  838. [
  839. f'"{self._get_full_import_name(tpi)}"'
  840. for tpi in first_party_imports[
  841. : int(MAX_NUMBER_OF_IMPORT_SHOWN // 2)
  842. ]
  843. ]
  844. )
  845. + " (...) "
  846. + ", ".join(
  847. [
  848. f'"{self._get_full_import_name(tpi)}"'
  849. for tpi in first_party_imports[
  850. int(-MAX_NUMBER_OF_IMPORT_SHOWN // 2) :
  851. ]
  852. ]
  853. )
  854. )
  855. else:
  856. imports_list = ", ".join(
  857. [
  858. f'"{self._get_full_import_name(fpi)}"'
  859. for fpi in first_party_imports
  860. ]
  861. )
  862. first_party = f"first party import{plural} {imports_list}"
  863. else:
  864. first_party = ""
  865. if local_imports:
  866. plural = "s" if len(local_imports) > 1 else ""
  867. if len(local_imports) > MAX_NUMBER_OF_IMPORT_SHOWN:
  868. imports_list = (
  869. ", ".join(
  870. [
  871. f'"{self._get_full_import_name(tpi)}"'
  872. for tpi in local_imports[
  873. : int(MAX_NUMBER_OF_IMPORT_SHOWN // 2)
  874. ]
  875. ]
  876. )
  877. + " (...) "
  878. + ", ".join(
  879. [
  880. f'"{self._get_full_import_name(tpi)}"'
  881. for tpi in local_imports[
  882. int(-MAX_NUMBER_OF_IMPORT_SHOWN // 2) :
  883. ]
  884. ]
  885. )
  886. )
  887. else:
  888. imports_list = ", ".join(
  889. [f'"{self._get_full_import_name(li)}"' for li in local_imports]
  890. )
  891. local = f"local import{plural} {imports_list}"
  892. else:
  893. local = ""
  894. delimiter_third_party = (
  895. (
  896. ", "
  897. if (first_party and local)
  898. else (" and " if (first_party or local) else "")
  899. )
  900. if third_party
  901. else ""
  902. )
  903. delimiter_first_party1 = (
  904. (", " if (third_party and local) else " ") if first_party else ""
  905. )
  906. delimiter_first_party2 = ("and " if local else "") if first_party else ""
  907. delimiter_first_party = f"{delimiter_first_party1}{delimiter_first_party2}"
  908. msg = (
  909. f"{third_party}{delimiter_third_party}"
  910. f"{first_party}{delimiter_first_party}"
  911. f'{local if local else ""}'
  912. )
  913. return msg
  914. def _get_full_import_name(self, importNode: ImportNode) -> str:
  915. # construct a more descriptive name of the import
  916. # for: import X, this returns X
  917. # for: import X.Y this returns X.Y
  918. # for: from X import Y, this returns X.Y
  919. try:
  920. # this will only succeed for ImportFrom nodes, which in themselves
  921. # contain the information needed to reconstruct the package
  922. return f"{importNode[0].modname}.{importNode[0].names[0][0]}"
  923. except AttributeError:
  924. # in all other cases, the import will either be X or X.Y
  925. node: str = importNode[0].names[0][0]
  926. package: str = importNode[1]
  927. if node.split(".")[0] == package:
  928. # this is sufficient with one import per line, since package = X
  929. # and node = X.Y or X
  930. return node
  931. # when there is a node that contains multiple imports, the "current"
  932. # import being analyzed is specified by package (node is the first
  933. # import on the line and therefore != package in this case)
  934. return package
  935. def _get_imported_module(
  936. self, importnode: ImportNode, modname: str
  937. ) -> nodes.Module | None:
  938. try:
  939. return importnode.do_import_module(modname)
  940. except astroid.TooManyLevelsError:
  941. if _ignore_import_failure(importnode, modname, self._ignored_modules):
  942. return None
  943. self.add_message("relative-beyond-top-level", node=importnode)
  944. except astroid.AstroidSyntaxError as exc:
  945. message = f"Cannot import {modname!r} due to '{exc.error}'"
  946. self.add_message(
  947. "syntax-error", line=importnode.lineno, args=message, confidence=HIGH
  948. )
  949. except astroid.AstroidBuildingError:
  950. if not self.linter.is_message_enabled("import-error"):
  951. return None
  952. if _ignore_import_failure(importnode, modname, self._ignored_modules):
  953. return None
  954. if (
  955. not self.linter.config.analyse_fallback_blocks
  956. and is_from_fallback_block(importnode)
  957. ):
  958. return None
  959. dotted_modname = get_import_name(importnode, modname)
  960. self.add_message("import-error", args=repr(dotted_modname), node=importnode)
  961. except Exception as e: # pragma: no cover
  962. raise astroid.AstroidError from e
  963. return None
  964. def _add_imported_module(self, node: ImportNode, importedmodname: str) -> None:
  965. """Notify an imported module, used to analyze dependencies."""
  966. module_file = node.root().file
  967. context_name = node.root().name
  968. base = os.path.splitext(os.path.basename(module_file))[0]
  969. try:
  970. if isinstance(node, nodes.ImportFrom) and node.level:
  971. importedmodname = astroid.modutils.get_module_part(
  972. importedmodname, module_file
  973. )
  974. else:
  975. importedmodname = astroid.modutils.get_module_part(importedmodname)
  976. except ImportError:
  977. pass
  978. if context_name == importedmodname:
  979. self.add_message("import-self", node=node)
  980. elif not astroid.modutils.is_stdlib_module(importedmodname):
  981. # if this is not a package __init__ module
  982. if base != "__init__" and context_name not in self._module_pkg:
  983. # record the module's parent, or the module itself if this is
  984. # a top level module, as the package it belongs to
  985. self._module_pkg[context_name] = context_name.rsplit(".", 1)[0]
  986. # handle dependencies
  987. dependencies_stat: dict[str, set[str]] = self.linter.stats.dependencies
  988. importedmodnames = dependencies_stat.setdefault(importedmodname, set())
  989. if context_name not in importedmodnames:
  990. importedmodnames.add(context_name)
  991. # update import graph
  992. self.import_graph[context_name].add(importedmodname)
  993. if not self.linter.is_message_enabled(
  994. "cyclic-import", line=node.lineno
  995. ) or in_type_checking_block(node):
  996. self._excluded_edges[context_name].add(importedmodname)
  997. def _check_preferred_module(self, node: ImportNode, mod_path: str) -> None:
  998. """Check if the module has a preferred replacement."""
  999. mod_compare = [mod_path]
  1000. # build a comparison list of possible names using importfrom
  1001. if isinstance(node, nodes.ImportFrom):
  1002. mod_compare = [f"{node.modname}.{name[0]}" for name in node.names]
  1003. # find whether there are matches with the import vs preferred_modules keys
  1004. matches = [
  1005. k
  1006. for k in self.preferred_modules
  1007. for mod in mod_compare
  1008. # exact match
  1009. if k == mod
  1010. # checks for base module matches
  1011. or k in mod.split(".")[0]
  1012. ]
  1013. # if we have matches, add message
  1014. if matches:
  1015. self.add_message(
  1016. "preferred-module",
  1017. node=node,
  1018. args=(self.preferred_modules[matches[0]], matches[0]),
  1019. )
  1020. def _check_import_as_rename(self, node: ImportNode) -> None:
  1021. names = node.names
  1022. for name in names:
  1023. if not all(name):
  1024. return
  1025. splitted_packages = name[0].rsplit(".", maxsplit=1)
  1026. import_name = splitted_packages[-1]
  1027. aliased_name = name[1]
  1028. if import_name != aliased_name:
  1029. continue
  1030. if len(splitted_packages) == 1 and (
  1031. self._allow_reexport_package is False
  1032. or self._current_module_package is False
  1033. ):
  1034. self.add_message("useless-import-alias", node=node, confidence=HIGH)
  1035. elif len(splitted_packages) == 2:
  1036. self.add_message(
  1037. "consider-using-from-import",
  1038. node=node,
  1039. args=(splitted_packages[0], import_name),
  1040. )
  1041. def _check_reimport(
  1042. self,
  1043. node: ImportNode,
  1044. basename: str | None = None,
  1045. level: int | None = None,
  1046. ) -> None:
  1047. """Check if a module with the same name is already imported or aliased."""
  1048. if not self.linter.is_message_enabled(
  1049. "reimported"
  1050. ) and not self.linter.is_message_enabled("shadowed-import"):
  1051. return
  1052. frame = node.frame()
  1053. root = node.root()
  1054. contexts = [(frame, level)]
  1055. if root is not frame:
  1056. contexts.append((root, None))
  1057. for known_context, known_level in contexts:
  1058. for name, alias in node.names:
  1059. first, msg = _get_first_import(
  1060. node, known_context, name, basename, known_level, alias
  1061. )
  1062. if first is not None and msg is not None:
  1063. name = name if msg == "reimported" else alias
  1064. self.add_message(
  1065. msg, node=node, args=(name, first.fromlineno), confidence=HIGH
  1066. )
  1067. def _report_external_dependencies(
  1068. self, sect: Section, _: LinterStats, _dummy: LinterStats | None
  1069. ) -> None:
  1070. """Return a verbatim layout for displaying dependencies."""
  1071. dep_info = _make_tree_defs(self._external_dependencies_info.items())
  1072. if not dep_info:
  1073. raise EmptyReportError()
  1074. tree_str = _repr_tree_defs(dep_info)
  1075. sect.append(VerbatimText(tree_str))
  1076. def _report_dependencies_graph(
  1077. self, sect: Section, _: LinterStats, _dummy: LinterStats | None
  1078. ) -> None:
  1079. """Write dependencies as a dot (graphviz) file."""
  1080. dep_info = self.linter.stats.dependencies
  1081. if not (
  1082. dep_info
  1083. and (
  1084. self.linter.config.import_graph
  1085. or self.linter.config.ext_import_graph
  1086. or self.linter.config.int_import_graph
  1087. )
  1088. ):
  1089. raise EmptyReportError()
  1090. filename = self.linter.config.import_graph
  1091. if filename:
  1092. _make_graph(filename, dep_info, sect, "")
  1093. filename = self.linter.config.ext_import_graph
  1094. if filename:
  1095. _make_graph(filename, self._external_dependencies_info, sect, "external ")
  1096. filename = self.linter.config.int_import_graph
  1097. if filename:
  1098. _make_graph(filename, self._internal_dependencies_info, sect, "internal ")
  1099. def _filter_dependencies_graph(self, internal: bool) -> defaultdict[str, set[str]]:
  1100. """Build the internal or the external dependency graph."""
  1101. graph: defaultdict[str, set[str]] = defaultdict(set)
  1102. for importee, importers in self.linter.stats.dependencies.items():
  1103. for importer in importers:
  1104. package = self._module_pkg.get(importer, importer)
  1105. is_inside = importee.startswith(package)
  1106. if (is_inside and internal) or (not is_inside and not internal):
  1107. graph[importee].add(importer)
  1108. return graph
  1109. @cached_property
  1110. def _external_dependencies_info(self) -> defaultdict[str, set[str]]:
  1111. """Return cached external dependencies information or build and
  1112. cache them.
  1113. """
  1114. return self._filter_dependencies_graph(internal=False)
  1115. @cached_property
  1116. def _internal_dependencies_info(self) -> defaultdict[str, set[str]]:
  1117. """Return cached internal dependencies information or build and
  1118. cache them.
  1119. """
  1120. return self._filter_dependencies_graph(internal=True)
  1121. def _check_wildcard_imports(
  1122. self, node: nodes.ImportFrom, imported_module: nodes.Module | None
  1123. ) -> None:
  1124. if node.root().package:
  1125. # Skip the check if in __init__.py issue #2026
  1126. return
  1127. wildcard_import_is_allowed = self._wildcard_import_is_allowed(imported_module)
  1128. for name, _ in node.names:
  1129. if name == "*" and not wildcard_import_is_allowed:
  1130. self.add_message("wildcard-import", args=node.modname, node=node)
  1131. def _wildcard_import_is_allowed(self, imported_module: nodes.Module | None) -> bool:
  1132. return (
  1133. self.linter.config.allow_wildcard_with_all
  1134. and imported_module is not None
  1135. and "__all__" in imported_module.locals
  1136. )
  1137. def _check_toplevel(self, node: ImportNode) -> None:
  1138. """Check whether the import is made outside the module toplevel."""
  1139. # If the scope of the import is a module, then obviously it is
  1140. # not outside the module toplevel.
  1141. if isinstance(node.scope(), nodes.Module):
  1142. return
  1143. module_names = [
  1144. (
  1145. f"{node.modname}.{name[0]}"
  1146. if isinstance(node, nodes.ImportFrom)
  1147. else name[0]
  1148. )
  1149. for name in node.names
  1150. ]
  1151. # Get the full names of all the imports that are only allowed at the module level
  1152. scoped_imports = [
  1153. name for name in module_names if name not in self._allow_any_import_level
  1154. ]
  1155. if scoped_imports:
  1156. self.add_message(
  1157. "import-outside-toplevel", args=", ".join(scoped_imports), node=node
  1158. )
  1159. def register(linter: PyLinter) -> None:
  1160. linter.register_checker(ImportsChecker(linter))