imports.py 41 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103
  1. # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
  2. # For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE
  3. # Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt
  4. """Imports checkers for Python code."""
  5. from __future__ import annotations
  6. import collections
  7. import copy
  8. import os
  9. import sys
  10. from collections import defaultdict
  11. from collections.abc import ItemsView, Sequence
  12. from typing import TYPE_CHECKING, Any, Dict, List, Union
  13. import astroid
  14. from astroid import nodes
  15. from astroid.nodes._base_nodes import ImportNode
  16. from pylint.checkers import BaseChecker, DeprecatedMixin
  17. from pylint.checkers.utils import (
  18. get_import_name,
  19. in_type_checking_block,
  20. is_from_fallback_block,
  21. is_sys_guard,
  22. node_ignores_exception,
  23. )
  24. from pylint.exceptions import EmptyReportError
  25. from pylint.graph import DotBackend, get_cycles
  26. from pylint.interfaces import HIGH
  27. from pylint.reporters.ureports.nodes import Paragraph, Section, VerbatimText
  28. from pylint.typing import MessageDefinitionTuple
  29. from pylint.utils import IsortDriver
  30. from pylint.utils.linterstats import LinterStats
  31. if TYPE_CHECKING:
  32. from pylint.lint import PyLinter
  33. if sys.version_info >= (3, 8):
  34. from functools import cached_property
  35. else:
  36. from astroid.decorators import cachedproperty as cached_property
  37. # The dictionary with Any should actually be a _ImportTree again
  38. # but mypy doesn't support recursive types yet
  39. _ImportTree = Dict[str, Union[List[Dict[str, Any]], List[str]]]
  40. DEPRECATED_MODULES = {
  41. (0, 0, 0): {"tkinter.tix", "fpectl"},
  42. (3, 2, 0): {"optparse"},
  43. (3, 3, 0): {"xml.etree.cElementTree"},
  44. (3, 4, 0): {"imp"},
  45. (3, 5, 0): {"formatter"},
  46. (3, 6, 0): {"asynchat", "asyncore", "smtpd"},
  47. (3, 7, 0): {"macpath"},
  48. (3, 9, 0): {"lib2to3", "parser", "symbol", "binhex"},
  49. (3, 10, 0): {"distutils", "typing.io", "typing.re"},
  50. (3, 11, 0): {
  51. "aifc",
  52. "audioop",
  53. "cgi",
  54. "cgitb",
  55. "chunk",
  56. "crypt",
  57. "imghdr",
  58. "msilib",
  59. "mailcap",
  60. "nis",
  61. "nntplib",
  62. "ossaudiodev",
  63. "pipes",
  64. "sndhdr",
  65. "spwd",
  66. "sunau",
  67. "sre_compile",
  68. "sre_constants",
  69. "sre_parse",
  70. "telnetlib",
  71. "uu",
  72. "xdrlib",
  73. },
  74. }
  75. def _qualified_names(modname: str | None) -> list[str]:
  76. """Split the names of the given module into subparts.
  77. For example,
  78. _qualified_names('pylint.checkers.ImportsChecker')
  79. returns
  80. ['pylint', 'pylint.checkers', 'pylint.checkers.ImportsChecker']
  81. """
  82. names = modname.split(".") if modname is not None else ""
  83. return [".".join(names[0 : i + 1]) for i in range(len(names))]
  84. def _get_first_import(
  85. node: ImportNode,
  86. context: nodes.LocalsDictNodeNG,
  87. name: str,
  88. base: str | None,
  89. level: int | None,
  90. alias: str | None,
  91. ) -> tuple[nodes.Import | nodes.ImportFrom | None, str | None]:
  92. """Return the node where [base.]<name> is imported or None if not found."""
  93. fullname = f"{base}.{name}" if base else name
  94. first = None
  95. found = False
  96. msg = "reimported"
  97. for first in context.body:
  98. if first is node:
  99. continue
  100. if first.scope() is node.scope() and first.fromlineno > node.fromlineno:
  101. continue
  102. if isinstance(first, nodes.Import):
  103. if any(fullname == iname[0] for iname in first.names):
  104. found = True
  105. break
  106. for imported_name, imported_alias in first.names:
  107. if not imported_alias and imported_name == alias:
  108. found = True
  109. msg = "shadowed-import"
  110. break
  111. if found:
  112. break
  113. elif isinstance(first, nodes.ImportFrom):
  114. if level == first.level:
  115. for imported_name, imported_alias in first.names:
  116. if fullname == f"{first.modname}.{imported_name}":
  117. found = True
  118. break
  119. if (
  120. name != "*"
  121. and name == imported_name
  122. and not (alias or imported_alias)
  123. ):
  124. found = True
  125. break
  126. if not imported_alias and imported_name == alias:
  127. found = True
  128. msg = "shadowed-import"
  129. break
  130. if found:
  131. break
  132. if found and not astroid.are_exclusive(first, node):
  133. return first, msg
  134. return None, None
  135. def _ignore_import_failure(
  136. node: ImportNode,
  137. modname: str | None,
  138. ignored_modules: Sequence[str],
  139. ) -> bool:
  140. for submodule in _qualified_names(modname):
  141. if submodule in ignored_modules:
  142. return True
  143. # Ignore import failure if part of guarded import block
  144. # I.e. `sys.version_info` or `typing.TYPE_CHECKING`
  145. if in_type_checking_block(node):
  146. return True
  147. if isinstance(node.parent, nodes.If) and is_sys_guard(node.parent):
  148. return True
  149. return node_ignores_exception(node, ImportError)
  150. # utilities to represents import dependencies as tree and dot graph ###########
  151. def _make_tree_defs(mod_files_list: ItemsView[str, set[str]]) -> _ImportTree:
  152. """Get a list of 2-uple (module, list_of_files_which_import_this_module),
  153. it will return a dictionary to represent this as a tree.
  154. """
  155. tree_defs: _ImportTree = {}
  156. for mod, files in mod_files_list:
  157. node: list[_ImportTree | list[str]] = [tree_defs, []]
  158. for prefix in mod.split("."):
  159. assert isinstance(node[0], dict)
  160. node = node[0].setdefault(prefix, ({}, [])) # type: ignore[arg-type,assignment]
  161. assert isinstance(node[1], list)
  162. node[1].extend(files)
  163. return tree_defs
  164. def _repr_tree_defs(data: _ImportTree, indent_str: str | None = None) -> str:
  165. """Return a string which represents imports as a tree."""
  166. lines = []
  167. nodes_items = data.items()
  168. for i, (mod, (sub, files)) in enumerate(sorted(nodes_items, key=lambda x: x[0])):
  169. files_list = "" if not files else f"({','.join(sorted(files))})"
  170. if indent_str is None:
  171. lines.append(f"{mod} {files_list}")
  172. sub_indent_str = " "
  173. else:
  174. lines.append(rf"{indent_str}\-{mod} {files_list}")
  175. if i == len(nodes_items) - 1:
  176. sub_indent_str = f"{indent_str} "
  177. else:
  178. sub_indent_str = f"{indent_str}| "
  179. if sub and isinstance(sub, dict):
  180. lines.append(_repr_tree_defs(sub, sub_indent_str))
  181. return "\n".join(lines)
  182. def _dependencies_graph(filename: str, dep_info: dict[str, set[str]]) -> str:
  183. """Write dependencies as a dot (graphviz) file."""
  184. done = {}
  185. printer = DotBackend(os.path.splitext(os.path.basename(filename))[0], rankdir="LR")
  186. printer.emit('URL="." node[shape="box"]')
  187. for modname, dependencies in sorted(dep_info.items()):
  188. sorted_dependencies = sorted(dependencies)
  189. done[modname] = 1
  190. printer.emit_node(modname)
  191. for depmodname in sorted_dependencies:
  192. if depmodname not in done:
  193. done[depmodname] = 1
  194. printer.emit_node(depmodname)
  195. for depmodname, dependencies in sorted(dep_info.items()):
  196. for modname in sorted(dependencies):
  197. printer.emit_edge(modname, depmodname)
  198. return printer.generate(filename)
  199. def _make_graph(
  200. filename: str, dep_info: dict[str, set[str]], sect: Section, gtype: str
  201. ) -> None:
  202. """Generate a dependencies graph and add some information about it in the
  203. report's section.
  204. """
  205. outputfile = _dependencies_graph(filename, dep_info)
  206. sect.append(Paragraph((f"{gtype}imports graph has been written to {outputfile}",)))
  207. # the import checker itself ###################################################
  208. MSGS: dict[str, MessageDefinitionTuple] = {
  209. "E0401": (
  210. "Unable to import %s",
  211. "import-error",
  212. "Used when pylint has been unable to import a module.",
  213. {"old_names": [("F0401", "old-import-error")]},
  214. ),
  215. "E0402": (
  216. "Attempted relative import beyond top-level package",
  217. "relative-beyond-top-level",
  218. "Used when a relative import tries to access too many levels "
  219. "in the current package.",
  220. ),
  221. "R0401": (
  222. "Cyclic import (%s)",
  223. "cyclic-import",
  224. "Used when a cyclic import between two or more modules is detected.",
  225. ),
  226. "R0402": (
  227. "Use 'from %s import %s' instead",
  228. "consider-using-from-import",
  229. "Emitted when a submodule of a package is imported and "
  230. "aliased with the same name, "
  231. "e.g., instead of ``import concurrent.futures as futures`` use "
  232. "``from concurrent import futures``.",
  233. ),
  234. "W0401": (
  235. "Wildcard import %s",
  236. "wildcard-import",
  237. "Used when `from module import *` is detected.",
  238. ),
  239. "W0404": (
  240. "Reimport %r (imported line %s)",
  241. "reimported",
  242. "Used when a module is imported more than once.",
  243. ),
  244. "W0406": (
  245. "Module import itself",
  246. "import-self",
  247. "Used when a module is importing itself.",
  248. ),
  249. "W0407": (
  250. "Prefer importing %r instead of %r",
  251. "preferred-module",
  252. "Used when a module imported has a preferred replacement module.",
  253. ),
  254. "W0410": (
  255. "__future__ import is not the first non docstring statement",
  256. "misplaced-future",
  257. "Python 2.5 and greater require __future__ import to be the "
  258. "first non docstring statement in the module.",
  259. ),
  260. "C0410": (
  261. "Multiple imports on one line (%s)",
  262. "multiple-imports",
  263. "Used when import statement importing multiple modules is detected.",
  264. ),
  265. "C0411": (
  266. "%s should be placed before %s",
  267. "wrong-import-order",
  268. "Used when PEP8 import order is not respected (standard imports "
  269. "first, then third-party libraries, then local imports).",
  270. ),
  271. "C0412": (
  272. "Imports from package %s are not grouped",
  273. "ungrouped-imports",
  274. "Used when imports are not grouped by packages.",
  275. ),
  276. "C0413": (
  277. 'Import "%s" should be placed at the top of the module',
  278. "wrong-import-position",
  279. "Used when code and imports are mixed.",
  280. ),
  281. "C0414": (
  282. "Import alias does not rename original package",
  283. "useless-import-alias",
  284. "Used when an import alias is same as original package, "
  285. "e.g., using import numpy as numpy instead of import numpy as np.",
  286. ),
  287. "C0415": (
  288. "Import outside toplevel (%s)",
  289. "import-outside-toplevel",
  290. "Used when an import statement is used anywhere other than the module "
  291. "toplevel. Move this import to the top of the file.",
  292. ),
  293. "W0416": (
  294. "Shadowed %r (imported line %s)",
  295. "shadowed-import",
  296. "Used when a module is aliased with a name that shadows another import.",
  297. ),
  298. }
  299. DEFAULT_STANDARD_LIBRARY = ()
  300. DEFAULT_KNOWN_THIRD_PARTY = ("enchant",)
  301. DEFAULT_PREFERRED_MODULES = ()
  302. class ImportsChecker(DeprecatedMixin, BaseChecker):
  303. """BaseChecker for import statements.
  304. Checks for
  305. * external modules dependencies
  306. * relative / wildcard imports
  307. * cyclic imports
  308. * uses of deprecated modules
  309. * uses of modules instead of preferred modules
  310. """
  311. name = "imports"
  312. msgs = {**DeprecatedMixin.DEPRECATED_MODULE_MESSAGE, **MSGS}
  313. default_deprecated_modules = ()
  314. options = (
  315. (
  316. "deprecated-modules",
  317. {
  318. "default": default_deprecated_modules,
  319. "type": "csv",
  320. "metavar": "<modules>",
  321. "help": "Deprecated modules which should not be used,"
  322. " separated by a comma.",
  323. },
  324. ),
  325. (
  326. "preferred-modules",
  327. {
  328. "default": DEFAULT_PREFERRED_MODULES,
  329. "type": "csv",
  330. "metavar": "<module:preferred-module>",
  331. "help": "Couples of modules and preferred modules,"
  332. " separated by a comma.",
  333. },
  334. ),
  335. (
  336. "import-graph",
  337. {
  338. "default": "",
  339. "type": "path",
  340. "metavar": "<file.gv>",
  341. "help": "Output a graph (.gv or any supported image format) of"
  342. " all (i.e. internal and external) dependencies to the given file"
  343. " (report RP0402 must not be disabled).",
  344. },
  345. ),
  346. (
  347. "ext-import-graph",
  348. {
  349. "default": "",
  350. "type": "path",
  351. "metavar": "<file.gv>",
  352. "help": "Output a graph (.gv or any supported image format)"
  353. " of external dependencies to the given file"
  354. " (report RP0402 must not be disabled).",
  355. },
  356. ),
  357. (
  358. "int-import-graph",
  359. {
  360. "default": "",
  361. "type": "path",
  362. "metavar": "<file.gv>",
  363. "help": "Output a graph (.gv or any supported image format)"
  364. " of internal dependencies to the given file"
  365. " (report RP0402 must not be disabled).",
  366. },
  367. ),
  368. (
  369. "known-standard-library",
  370. {
  371. "default": DEFAULT_STANDARD_LIBRARY,
  372. "type": "csv",
  373. "metavar": "<modules>",
  374. "help": "Force import order to recognize a module as part of "
  375. "the standard compatibility libraries.",
  376. },
  377. ),
  378. (
  379. "known-third-party",
  380. {
  381. "default": DEFAULT_KNOWN_THIRD_PARTY,
  382. "type": "csv",
  383. "metavar": "<modules>",
  384. "help": "Force import order to recognize a module as part of "
  385. "a third party library.",
  386. },
  387. ),
  388. (
  389. "allow-any-import-level",
  390. {
  391. "default": (),
  392. "type": "csv",
  393. "metavar": "<modules>",
  394. "help": (
  395. "List of modules that can be imported at any level, not just "
  396. "the top level one."
  397. ),
  398. },
  399. ),
  400. (
  401. "allow-wildcard-with-all",
  402. {
  403. "default": False,
  404. "type": "yn",
  405. "metavar": "<y or n>",
  406. "help": "Allow wildcard imports from modules that define __all__.",
  407. },
  408. ),
  409. (
  410. "allow-reexport-from-package",
  411. {
  412. "default": False,
  413. "type": "yn",
  414. "metavar": "<y or n>",
  415. "help": "Allow explicit reexports by alias from a package __init__.",
  416. },
  417. ),
  418. )
  419. def __init__(self, linter: PyLinter) -> None:
  420. BaseChecker.__init__(self, linter)
  421. self.import_graph: defaultdict[str, set[str]] = defaultdict(set)
  422. self._imports_stack: list[tuple[ImportNode, str]] = []
  423. self._first_non_import_node = None
  424. self._module_pkg: dict[
  425. Any, Any
  426. ] = {} # mapping of modules to the pkg they belong in
  427. self._allow_any_import_level: set[Any] = set()
  428. self.reports = (
  429. ("RP0401", "External dependencies", self._report_external_dependencies),
  430. ("RP0402", "Modules dependencies graph", self._report_dependencies_graph),
  431. )
  432. def open(self) -> None:
  433. """Called before visiting project (i.e set of modules)."""
  434. self.linter.stats.dependencies = {}
  435. self.linter.stats = self.linter.stats
  436. self.import_graph = defaultdict(set)
  437. self._module_pkg = {} # mapping of modules to the pkg they belong in
  438. self._current_module_package = False
  439. self._excluded_edges: defaultdict[str, set[str]] = defaultdict(set)
  440. self._ignored_modules: Sequence[str] = self.linter.config.ignored_modules
  441. # Build a mapping {'module': 'preferred-module'}
  442. self.preferred_modules = dict(
  443. module.split(":")
  444. for module in self.linter.config.preferred_modules
  445. if ":" in module
  446. )
  447. self._allow_any_import_level = set(self.linter.config.allow_any_import_level)
  448. self._allow_reexport_package = self.linter.config.allow_reexport_from_package
  449. def _import_graph_without_ignored_edges(self) -> defaultdict[str, set[str]]:
  450. filtered_graph = copy.deepcopy(self.import_graph)
  451. for node in filtered_graph:
  452. filtered_graph[node].difference_update(self._excluded_edges[node])
  453. return filtered_graph
  454. def close(self) -> None:
  455. """Called before visiting project (i.e set of modules)."""
  456. if self.linter.is_message_enabled("cyclic-import"):
  457. graph = self._import_graph_without_ignored_edges()
  458. vertices = list(graph)
  459. for cycle in get_cycles(graph, vertices=vertices):
  460. self.add_message("cyclic-import", args=" -> ".join(cycle))
  461. def deprecated_modules(self) -> set[str]:
  462. """Callback returning the deprecated modules."""
  463. # First get the modules the user indicated
  464. all_deprecated_modules = set(self.linter.config.deprecated_modules)
  465. # Now get the hard-coded ones from the stdlib
  466. for since_vers, mod_set in DEPRECATED_MODULES.items():
  467. if since_vers <= sys.version_info:
  468. all_deprecated_modules = all_deprecated_modules.union(mod_set)
  469. return all_deprecated_modules
  470. def visit_module(self, node: nodes.Module) -> None:
  471. """Store if current module is a package, i.e. an __init__ file."""
  472. self._current_module_package = node.package
  473. def visit_import(self, node: nodes.Import) -> None:
  474. """Triggered when an import statement is seen."""
  475. self._check_reimport(node)
  476. self._check_import_as_rename(node)
  477. self._check_toplevel(node)
  478. names = [name for name, _ in node.names]
  479. if len(names) >= 2:
  480. self.add_message("multiple-imports", args=", ".join(names), node=node)
  481. for name in names:
  482. self.check_deprecated_module(node, name)
  483. self._check_preferred_module(node, name)
  484. imported_module = self._get_imported_module(node, name)
  485. if isinstance(node.parent, nodes.Module):
  486. # Allow imports nested
  487. self._check_position(node)
  488. if isinstance(node.scope(), nodes.Module):
  489. self._record_import(node, imported_module)
  490. if imported_module is None:
  491. continue
  492. self._add_imported_module(node, imported_module.name)
  493. def visit_importfrom(self, node: nodes.ImportFrom) -> None:
  494. """Triggered when a from statement is seen."""
  495. basename = node.modname
  496. imported_module = self._get_imported_module(node, basename)
  497. absolute_name = get_import_name(node, basename)
  498. self._check_import_as_rename(node)
  499. self._check_misplaced_future(node)
  500. self.check_deprecated_module(node, absolute_name)
  501. self._check_preferred_module(node, basename)
  502. self._check_wildcard_imports(node, imported_module)
  503. self._check_same_line_imports(node)
  504. self._check_reimport(node, basename=basename, level=node.level)
  505. self._check_toplevel(node)
  506. if isinstance(node.parent, nodes.Module):
  507. # Allow imports nested
  508. self._check_position(node)
  509. if isinstance(node.scope(), nodes.Module):
  510. self._record_import(node, imported_module)
  511. if imported_module is None:
  512. return
  513. for name, _ in node.names:
  514. if name != "*":
  515. self._add_imported_module(node, f"{imported_module.name}.{name}")
  516. else:
  517. self._add_imported_module(node, imported_module.name)
  518. def leave_module(self, node: nodes.Module) -> None:
  519. # Check imports are grouped by category (standard, 3rd party, local)
  520. std_imports, ext_imports, loc_imports = self._check_imports_order(node)
  521. # Check that imports are grouped by package within a given category
  522. met_import: set[str] = set() # set for 'import x' style
  523. met_from: set[str] = set() # set for 'from x import y' style
  524. current_package = None
  525. for import_node, import_name in std_imports + ext_imports + loc_imports:
  526. met = met_from if isinstance(import_node, nodes.ImportFrom) else met_import
  527. package, _, _ = import_name.partition(".")
  528. if (
  529. current_package
  530. and current_package != package
  531. and package in met
  532. and not in_type_checking_block(import_node)
  533. and not (
  534. isinstance(import_node.parent, nodes.If)
  535. and is_sys_guard(import_node.parent)
  536. )
  537. ):
  538. self.add_message("ungrouped-imports", node=import_node, args=package)
  539. current_package = package
  540. if not self.linter.is_message_enabled(
  541. "ungrouped-imports", import_node.fromlineno
  542. ):
  543. continue
  544. met.add(package)
  545. self._imports_stack = []
  546. self._first_non_import_node = None
  547. def compute_first_non_import_node(
  548. self,
  549. node: nodes.If
  550. | nodes.Expr
  551. | nodes.Comprehension
  552. | nodes.IfExp
  553. | nodes.Assign
  554. | nodes.AssignAttr
  555. | nodes.TryExcept
  556. | nodes.TryFinally,
  557. ) -> None:
  558. # if the node does not contain an import instruction, and if it is the
  559. # first node of the module, keep a track of it (all the import positions
  560. # of the module will be compared to the position of this first
  561. # instruction)
  562. if self._first_non_import_node:
  563. return
  564. if not isinstance(node.parent, nodes.Module):
  565. return
  566. nested_allowed = [nodes.TryExcept, nodes.TryFinally]
  567. is_nested_allowed = [
  568. allowed for allowed in nested_allowed if isinstance(node, allowed)
  569. ]
  570. if is_nested_allowed and any(
  571. node.nodes_of_class((nodes.Import, nodes.ImportFrom))
  572. ):
  573. return
  574. if isinstance(node, nodes.Assign):
  575. # Add compatibility for module level dunder names
  576. # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
  577. valid_targets = [
  578. isinstance(target, nodes.AssignName)
  579. and target.name.startswith("__")
  580. and target.name.endswith("__")
  581. for target in node.targets
  582. ]
  583. if all(valid_targets):
  584. return
  585. self._first_non_import_node = node
  586. visit_tryfinally = (
  587. visit_tryexcept
  588. ) = (
  589. visit_assignattr
  590. ) = (
  591. visit_assign
  592. ) = (
  593. visit_ifexp
  594. ) = visit_comprehension = visit_expr = visit_if = compute_first_non_import_node
  595. def visit_functiondef(
  596. self, node: nodes.FunctionDef | nodes.While | nodes.For | nodes.ClassDef
  597. ) -> None:
  598. # If it is the first non import instruction of the module, record it.
  599. if self._first_non_import_node:
  600. return
  601. # Check if the node belongs to an `If` or a `Try` block. If they
  602. # contain imports, skip recording this node.
  603. if not isinstance(node.parent.scope(), nodes.Module):
  604. return
  605. root = node
  606. while not isinstance(root.parent, nodes.Module):
  607. root = root.parent
  608. if isinstance(root, (nodes.If, nodes.TryFinally, nodes.TryExcept)):
  609. if any(root.nodes_of_class((nodes.Import, nodes.ImportFrom))):
  610. return
  611. self._first_non_import_node = node
  612. visit_classdef = visit_for = visit_while = visit_functiondef
  613. def _check_misplaced_future(self, node: nodes.ImportFrom) -> None:
  614. basename = node.modname
  615. if basename == "__future__":
  616. # check if this is the first non-docstring statement in the module
  617. prev = node.previous_sibling()
  618. if prev:
  619. # consecutive future statements are possible
  620. if not (
  621. isinstance(prev, nodes.ImportFrom) and prev.modname == "__future__"
  622. ):
  623. self.add_message("misplaced-future", node=node)
  624. return
  625. def _check_same_line_imports(self, node: nodes.ImportFrom) -> None:
  626. # Detect duplicate imports on the same line.
  627. names = (name for name, _ in node.names)
  628. counter = collections.Counter(names)
  629. for name, count in counter.items():
  630. if count > 1:
  631. self.add_message("reimported", node=node, args=(name, node.fromlineno))
  632. def _check_position(self, node: ImportNode) -> None:
  633. """Check `node` import or importfrom node position is correct.
  634. Send a message if `node` comes before another instruction
  635. """
  636. # if a first non-import instruction has already been encountered,
  637. # it means the import comes after it and therefore is not well placed
  638. if self._first_non_import_node:
  639. if self.linter.is_message_enabled(
  640. "wrong-import-position", self._first_non_import_node.fromlineno
  641. ):
  642. self.add_message(
  643. "wrong-import-position", node=node, args=node.as_string()
  644. )
  645. else:
  646. self.linter.add_ignored_message(
  647. "wrong-import-position", node.fromlineno, node
  648. )
  649. def _record_import(
  650. self,
  651. node: ImportNode,
  652. importedmodnode: nodes.Module | None,
  653. ) -> None:
  654. """Record the package `node` imports from."""
  655. if isinstance(node, nodes.ImportFrom):
  656. importedname = node.modname
  657. else:
  658. importedname = importedmodnode.name if importedmodnode else None
  659. if not importedname:
  660. importedname = node.names[0][0].split(".")[0]
  661. if isinstance(node, nodes.ImportFrom) and (node.level or 0) >= 1:
  662. # We need the importedname with first point to detect local package
  663. # Example of node:
  664. # 'from .my_package1 import MyClass1'
  665. # the output should be '.my_package1' instead of 'my_package1'
  666. # Example of node:
  667. # 'from . import my_package2'
  668. # the output should be '.my_package2' instead of '{pyfile}'
  669. importedname = "." + importedname
  670. self._imports_stack.append((node, importedname))
  671. @staticmethod
  672. def _is_fallback_import(
  673. node: ImportNode, imports: list[tuple[ImportNode, str]]
  674. ) -> bool:
  675. imports = [import_node for (import_node, _) in imports]
  676. return any(astroid.are_exclusive(import_node, node) for import_node in imports)
  677. # pylint: disable = too-many-statements
  678. def _check_imports_order(
  679. self, _module_node: nodes.Module
  680. ) -> tuple[
  681. list[tuple[ImportNode, str]],
  682. list[tuple[ImportNode, str]],
  683. list[tuple[ImportNode, str]],
  684. ]:
  685. """Checks imports of module `node` are grouped by category.
  686. Imports must follow this order: standard, 3rd party, local
  687. """
  688. std_imports: list[tuple[ImportNode, str]] = []
  689. third_party_imports: list[tuple[ImportNode, str]] = []
  690. first_party_imports: list[tuple[ImportNode, str]] = []
  691. # need of a list that holds third or first party ordered import
  692. external_imports: list[tuple[ImportNode, str]] = []
  693. local_imports: list[tuple[ImportNode, str]] = []
  694. third_party_not_ignored: list[tuple[ImportNode, str]] = []
  695. first_party_not_ignored: list[tuple[ImportNode, str]] = []
  696. local_not_ignored: list[tuple[ImportNode, str]] = []
  697. isort_driver = IsortDriver(self.linter.config)
  698. for node, modname in self._imports_stack:
  699. if modname.startswith("."):
  700. package = "." + modname.split(".")[1]
  701. else:
  702. package = modname.split(".")[0]
  703. nested = not isinstance(node.parent, nodes.Module)
  704. ignore_for_import_order = not self.linter.is_message_enabled(
  705. "wrong-import-order", node.fromlineno
  706. )
  707. import_category = isort_driver.place_module(package)
  708. node_and_package_import = (node, package)
  709. if import_category in {"FUTURE", "STDLIB"}:
  710. std_imports.append(node_and_package_import)
  711. wrong_import = (
  712. third_party_not_ignored
  713. or first_party_not_ignored
  714. or local_not_ignored
  715. )
  716. if self._is_fallback_import(node, wrong_import):
  717. continue
  718. if wrong_import and not nested:
  719. self.add_message(
  720. "wrong-import-order",
  721. node=node,
  722. args=(
  723. f'standard import "{node.as_string()}"',
  724. f'"{wrong_import[0][0].as_string()}"',
  725. ),
  726. )
  727. elif import_category == "THIRDPARTY":
  728. third_party_imports.append(node_and_package_import)
  729. external_imports.append(node_and_package_import)
  730. if not nested:
  731. if not ignore_for_import_order:
  732. third_party_not_ignored.append(node_and_package_import)
  733. else:
  734. self.linter.add_ignored_message(
  735. "wrong-import-order", node.fromlineno, node
  736. )
  737. wrong_import = first_party_not_ignored or local_not_ignored
  738. if wrong_import and not nested:
  739. self.add_message(
  740. "wrong-import-order",
  741. node=node,
  742. args=(
  743. f'third party import "{node.as_string()}"',
  744. f'"{wrong_import[0][0].as_string()}"',
  745. ),
  746. )
  747. elif import_category == "FIRSTPARTY":
  748. first_party_imports.append(node_and_package_import)
  749. external_imports.append(node_and_package_import)
  750. if not nested:
  751. if not ignore_for_import_order:
  752. first_party_not_ignored.append(node_and_package_import)
  753. else:
  754. self.linter.add_ignored_message(
  755. "wrong-import-order", node.fromlineno, node
  756. )
  757. wrong_import = local_not_ignored
  758. if wrong_import and not nested:
  759. self.add_message(
  760. "wrong-import-order",
  761. node=node,
  762. args=(
  763. f'first party import "{node.as_string()}"',
  764. f'"{wrong_import[0][0].as_string()}"',
  765. ),
  766. )
  767. elif import_category == "LOCALFOLDER":
  768. local_imports.append((node, package))
  769. if not nested:
  770. if not ignore_for_import_order:
  771. local_not_ignored.append((node, package))
  772. else:
  773. self.linter.add_ignored_message(
  774. "wrong-import-order", node.fromlineno, node
  775. )
  776. return std_imports, external_imports, local_imports
  777. def _get_imported_module(
  778. self, importnode: ImportNode, modname: str | None
  779. ) -> nodes.Module | None:
  780. try:
  781. return importnode.do_import_module(modname)
  782. except astroid.TooManyLevelsError:
  783. if _ignore_import_failure(importnode, modname, self._ignored_modules):
  784. return None
  785. self.add_message("relative-beyond-top-level", node=importnode)
  786. except astroid.AstroidSyntaxError as exc:
  787. message = f"Cannot import {modname!r} due to '{exc.error}'"
  788. self.add_message(
  789. "syntax-error", line=importnode.lineno, args=message, confidence=HIGH
  790. )
  791. except astroid.AstroidBuildingError:
  792. if not self.linter.is_message_enabled("import-error"):
  793. return None
  794. if _ignore_import_failure(importnode, modname, self._ignored_modules):
  795. return None
  796. if (
  797. not self.linter.config.analyse_fallback_blocks
  798. and is_from_fallback_block(importnode)
  799. ):
  800. return None
  801. dotted_modname = get_import_name(importnode, modname)
  802. self.add_message("import-error", args=repr(dotted_modname), node=importnode)
  803. except Exception as e: # pragma: no cover
  804. raise astroid.AstroidError from e
  805. return None
  806. def _add_imported_module(self, node: ImportNode, importedmodname: str) -> None:
  807. """Notify an imported module, used to analyze dependencies."""
  808. module_file = node.root().file
  809. context_name = node.root().name
  810. base = os.path.splitext(os.path.basename(module_file))[0]
  811. try:
  812. importedmodname = astroid.modutils.get_module_part(
  813. importedmodname, module_file
  814. )
  815. except ImportError:
  816. pass
  817. if context_name == importedmodname:
  818. self.add_message("import-self", node=node)
  819. elif not astroid.modutils.is_stdlib_module(importedmodname):
  820. # if this is not a package __init__ module
  821. if base != "__init__" and context_name not in self._module_pkg:
  822. # record the module's parent, or the module itself if this is
  823. # a top level module, as the package it belongs to
  824. self._module_pkg[context_name] = context_name.rsplit(".", 1)[0]
  825. # handle dependencies
  826. dependencies_stat: dict[str, set[str]] = self.linter.stats.dependencies
  827. importedmodnames = dependencies_stat.setdefault(importedmodname, set())
  828. if context_name not in importedmodnames:
  829. importedmodnames.add(context_name)
  830. # update import graph
  831. self.import_graph[context_name].add(importedmodname)
  832. if not self.linter.is_message_enabled(
  833. "cyclic-import", line=node.lineno
  834. ) or in_type_checking_block(node):
  835. self._excluded_edges[context_name].add(importedmodname)
  836. def _check_preferred_module(self, node: ImportNode, mod_path: str) -> None:
  837. """Check if the module has a preferred replacement."""
  838. mod_compare = [mod_path]
  839. # build a comparison list of possible names using importfrom
  840. if isinstance(node, astroid.nodes.node_classes.ImportFrom):
  841. mod_compare = [f"{node.modname}.{name[0]}" for name in node.names]
  842. # find whether there are matches with the import vs preferred_modules keys
  843. matches = [
  844. k
  845. for k in self.preferred_modules
  846. for mod in mod_compare
  847. # exact match
  848. if k == mod
  849. # checks for base module matches
  850. or k in mod.split(".")[0]
  851. ]
  852. # if we have matches, add message
  853. if matches:
  854. self.add_message(
  855. "preferred-module",
  856. node=node,
  857. args=(self.preferred_modules[matches[0]], matches[0]),
  858. )
  859. def _check_import_as_rename(self, node: ImportNode) -> None:
  860. names = node.names
  861. for name in names:
  862. if not all(name):
  863. return
  864. splitted_packages = name[0].rsplit(".", maxsplit=1)
  865. import_name = splitted_packages[-1]
  866. aliased_name = name[1]
  867. if import_name != aliased_name:
  868. continue
  869. if len(splitted_packages) == 1 and (
  870. self._allow_reexport_package is False
  871. or self._current_module_package is False
  872. ):
  873. self.add_message("useless-import-alias", node=node, confidence=HIGH)
  874. elif len(splitted_packages) == 2:
  875. self.add_message(
  876. "consider-using-from-import",
  877. node=node,
  878. args=(splitted_packages[0], import_name),
  879. )
  880. def _check_reimport(
  881. self,
  882. node: ImportNode,
  883. basename: str | None = None,
  884. level: int | None = None,
  885. ) -> None:
  886. """Check if a module with the same name is already imported or aliased."""
  887. if not self.linter.is_message_enabled(
  888. "reimported"
  889. ) and not self.linter.is_message_enabled("shadowed-import"):
  890. return
  891. frame = node.frame(future=True)
  892. root = node.root()
  893. contexts = [(frame, level)]
  894. if root is not frame:
  895. contexts.append((root, None))
  896. for known_context, known_level in contexts:
  897. for name, alias in node.names:
  898. first, msg = _get_first_import(
  899. node, known_context, name, basename, known_level, alias
  900. )
  901. if first is not None and msg is not None:
  902. name = name if msg == "reimported" else alias
  903. self.add_message(
  904. msg, node=node, args=(name, first.fromlineno), confidence=HIGH
  905. )
  906. def _report_external_dependencies(
  907. self, sect: Section, _: LinterStats, _dummy: LinterStats | None
  908. ) -> None:
  909. """Return a verbatim layout for displaying dependencies."""
  910. dep_info = _make_tree_defs(self._external_dependencies_info.items())
  911. if not dep_info:
  912. raise EmptyReportError()
  913. tree_str = _repr_tree_defs(dep_info)
  914. sect.append(VerbatimText(tree_str))
  915. def _report_dependencies_graph(
  916. self, sect: Section, _: LinterStats, _dummy: LinterStats | None
  917. ) -> None:
  918. """Write dependencies as a dot (graphviz) file."""
  919. dep_info = self.linter.stats.dependencies
  920. if not dep_info or not (
  921. self.linter.config.import_graph
  922. or self.linter.config.ext_import_graph
  923. or self.linter.config.int_import_graph
  924. ):
  925. raise EmptyReportError()
  926. filename = self.linter.config.import_graph
  927. if filename:
  928. _make_graph(filename, dep_info, sect, "")
  929. filename = self.linter.config.ext_import_graph
  930. if filename:
  931. _make_graph(filename, self._external_dependencies_info, sect, "external ")
  932. filename = self.linter.config.int_import_graph
  933. if filename:
  934. _make_graph(filename, self._internal_dependencies_info, sect, "internal ")
  935. def _filter_dependencies_graph(self, internal: bool) -> defaultdict[str, set[str]]:
  936. """Build the internal or the external dependency graph."""
  937. graph: defaultdict[str, set[str]] = defaultdict(set)
  938. for importee, importers in self.linter.stats.dependencies.items():
  939. for importer in importers:
  940. package = self._module_pkg.get(importer, importer)
  941. is_inside = importee.startswith(package)
  942. if is_inside and internal or not is_inside and not internal:
  943. graph[importee].add(importer)
  944. return graph
  945. @cached_property
  946. def _external_dependencies_info(self) -> defaultdict[str, set[str]]:
  947. """Return cached external dependencies information or build and
  948. cache them.
  949. """
  950. return self._filter_dependencies_graph(internal=False)
  951. @cached_property
  952. def _internal_dependencies_info(self) -> defaultdict[str, set[str]]:
  953. """Return cached internal dependencies information or build and
  954. cache them.
  955. """
  956. return self._filter_dependencies_graph(internal=True)
  957. def _check_wildcard_imports(
  958. self, node: nodes.ImportFrom, imported_module: nodes.Module | None
  959. ) -> None:
  960. if node.root().package:
  961. # Skip the check if in __init__.py issue #2026
  962. return
  963. wildcard_import_is_allowed = self._wildcard_import_is_allowed(imported_module)
  964. for name, _ in node.names:
  965. if name == "*" and not wildcard_import_is_allowed:
  966. self.add_message("wildcard-import", args=node.modname, node=node)
  967. def _wildcard_import_is_allowed(self, imported_module: nodes.Module | None) -> bool:
  968. return (
  969. self.linter.config.allow_wildcard_with_all
  970. and imported_module is not None
  971. and "__all__" in imported_module.locals
  972. )
  973. def _check_toplevel(self, node: ImportNode) -> None:
  974. """Check whether the import is made outside the module toplevel."""
  975. # If the scope of the import is a module, then obviously it is
  976. # not outside the module toplevel.
  977. if isinstance(node.scope(), nodes.Module):
  978. return
  979. module_names = [
  980. f"{node.modname}.{name[0]}"
  981. if isinstance(node, nodes.ImportFrom)
  982. else name[0]
  983. for name in node.names
  984. ]
  985. # Get the full names of all the imports that are only allowed at the module level
  986. scoped_imports = [
  987. name for name in module_names if name not in self._allow_any_import_level
  988. ]
  989. if scoped_imports:
  990. self.add_message(
  991. "import-outside-toplevel", args=", ".join(scoped_imports), node=node
  992. )
  993. def register(linter: PyLinter) -> None:
  994. linter.register_checker(ImportsChecker(linter))