utils.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465
  1. # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
  2. # For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE
  3. # Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt
  4. from __future__ import annotations
  5. try:
  6. import isort.api
  7. import isort.settings
  8. HAS_ISORT_5 = True
  9. except ImportError: # isort < 5
  10. import isort
  11. HAS_ISORT_5 = False
  12. import argparse
  13. import codecs
  14. import os
  15. import re
  16. import sys
  17. import textwrap
  18. import tokenize
  19. import warnings
  20. from collections import deque
  21. from collections.abc import Iterable, Sequence
  22. from io import BufferedReader, BytesIO
  23. from typing import (
  24. TYPE_CHECKING,
  25. Any,
  26. List,
  27. Pattern,
  28. TextIO,
  29. Tuple,
  30. TypeVar,
  31. Union,
  32. overload,
  33. )
  34. from astroid import Module, modutils, nodes
  35. from pylint.constants import PY_EXTS
  36. from pylint.typing import OptionDict
  37. if sys.version_info >= (3, 8):
  38. from typing import Literal
  39. else:
  40. from typing_extensions import Literal
  41. if TYPE_CHECKING:
  42. from pylint.checkers.base_checker import BaseChecker
  43. from pylint.lint import PyLinter
  44. DEFAULT_LINE_LENGTH = 79
  45. # These are types used to overload get_global_option() and refer to the options type
  46. GLOBAL_OPTION_BOOL = Literal[
  47. "suggestion-mode",
  48. "analyse-fallback-blocks",
  49. "allow-global-unused-variables",
  50. ]
  51. GLOBAL_OPTION_INT = Literal["max-line-length", "docstring-min-length"]
  52. GLOBAL_OPTION_LIST = Literal["ignored-modules"]
  53. GLOBAL_OPTION_PATTERN = Literal[
  54. "no-docstring-rgx",
  55. "dummy-variables-rgx",
  56. "ignored-argument-names",
  57. "mixin-class-rgx",
  58. ]
  59. GLOBAL_OPTION_PATTERN_LIST = Literal["exclude-too-few-public-methods", "ignore-paths"]
  60. GLOBAL_OPTION_TUPLE_INT = Literal["py-version"]
  61. GLOBAL_OPTION_NAMES = Union[
  62. GLOBAL_OPTION_BOOL,
  63. GLOBAL_OPTION_INT,
  64. GLOBAL_OPTION_LIST,
  65. GLOBAL_OPTION_PATTERN,
  66. GLOBAL_OPTION_PATTERN_LIST,
  67. GLOBAL_OPTION_TUPLE_INT,
  68. ]
  69. T_GlobalOptionReturnTypes = TypeVar(
  70. "T_GlobalOptionReturnTypes",
  71. bool,
  72. int,
  73. List[str],
  74. Pattern[str],
  75. List[Pattern[str]],
  76. Tuple[int, ...],
  77. )
  78. def normalize_text(
  79. text: str, line_len: int = DEFAULT_LINE_LENGTH, indent: str = ""
  80. ) -> str:
  81. """Wrap the text on the given line length."""
  82. return "\n".join(
  83. textwrap.wrap(
  84. text, width=line_len, initial_indent=indent, subsequent_indent=indent
  85. )
  86. )
  87. CMPS = ["=", "-", "+"]
  88. # py3k has no more cmp builtin
  89. def cmp(a: int | float, b: int | float) -> int:
  90. return (a > b) - (a < b)
  91. def diff_string(old: int | float, new: int | float) -> str:
  92. """Given an old and new int value, return a string representing the
  93. difference.
  94. """
  95. diff = abs(old - new)
  96. diff_str = f"{CMPS[cmp(old, new)]}{diff and f'{diff:.2f}' or ''}"
  97. return diff_str
  98. def get_module_and_frameid(node: nodes.NodeNG) -> tuple[str, str]:
  99. """Return the module name and the frame id in the module."""
  100. frame = node.frame(future=True)
  101. module, obj = "", []
  102. while frame:
  103. if isinstance(frame, Module):
  104. module = frame.name
  105. else:
  106. obj.append(getattr(frame, "name", "<lambda>"))
  107. try:
  108. frame = frame.parent.frame(future=True)
  109. except AttributeError:
  110. break
  111. obj.reverse()
  112. return module, ".".join(obj)
  113. def get_rst_title(title: str, character: str) -> str:
  114. """Permit to get a title formatted as ReStructuredText test (underlined with a
  115. chosen character).
  116. """
  117. return f"{title}\n{character * len(title)}\n"
  118. def get_rst_section(
  119. section: str | None,
  120. options: list[tuple[str, OptionDict, Any]],
  121. doc: str | None = None,
  122. ) -> str:
  123. """Format an option's section using as a ReStructuredText formatted output."""
  124. result = ""
  125. if section:
  126. result += get_rst_title(section, "'")
  127. if doc:
  128. formatted_doc = normalize_text(doc)
  129. result += f"{formatted_doc}\n\n"
  130. for optname, optdict, value in options:
  131. help_opt = optdict.get("help")
  132. result += f":{optname}:\n"
  133. if help_opt:
  134. assert isinstance(help_opt, str)
  135. formatted_help = normalize_text(help_opt, indent=" ")
  136. result += f"{formatted_help}\n"
  137. if value and optname != "py-version":
  138. value = str(_format_option_value(optdict, value))
  139. result += f"\n Default: ``{value.replace('`` ', '```` ``')}``\n"
  140. return result
  141. def decoding_stream(
  142. stream: BufferedReader | BytesIO,
  143. encoding: str,
  144. errors: Literal["strict"] = "strict",
  145. ) -> codecs.StreamReader:
  146. try:
  147. reader_cls = codecs.getreader(encoding or sys.getdefaultencoding())
  148. except LookupError:
  149. reader_cls = codecs.getreader(sys.getdefaultencoding())
  150. return reader_cls(stream, errors)
  151. def tokenize_module(node: nodes.Module) -> list[tokenize.TokenInfo]:
  152. with node.stream() as stream:
  153. readline = stream.readline
  154. return list(tokenize.tokenize(readline))
  155. def register_plugins(linter: PyLinter, directory: str) -> None:
  156. """Load all module and package in the given directory, looking for a
  157. 'register' function in each one, used to register pylint checkers.
  158. """
  159. imported = {}
  160. for filename in os.listdir(directory):
  161. base, extension = os.path.splitext(filename)
  162. if base in imported or base == "__pycache__":
  163. continue
  164. if (
  165. extension in PY_EXTS
  166. and base != "__init__"
  167. or (
  168. not extension
  169. and os.path.isdir(os.path.join(directory, base))
  170. and not filename.startswith(".")
  171. )
  172. ):
  173. try:
  174. module = modutils.load_module_from_file(
  175. os.path.join(directory, filename)
  176. )
  177. except ValueError:
  178. # empty module name (usually Emacs auto-save files)
  179. continue
  180. except ImportError as exc:
  181. print(f"Problem importing module {filename}: {exc}", file=sys.stderr)
  182. else:
  183. if hasattr(module, "register"):
  184. module.register(linter)
  185. imported[base] = 1
  186. @overload
  187. def get_global_option(
  188. checker: BaseChecker, option: GLOBAL_OPTION_BOOL, default: bool | None = ...
  189. ) -> bool:
  190. ...
  191. @overload
  192. def get_global_option(
  193. checker: BaseChecker, option: GLOBAL_OPTION_INT, default: int | None = ...
  194. ) -> int:
  195. ...
  196. @overload
  197. def get_global_option(
  198. checker: BaseChecker,
  199. option: GLOBAL_OPTION_LIST,
  200. default: list[str] | None = ...,
  201. ) -> list[str]:
  202. ...
  203. @overload
  204. def get_global_option(
  205. checker: BaseChecker,
  206. option: GLOBAL_OPTION_PATTERN,
  207. default: Pattern[str] | None = ...,
  208. ) -> Pattern[str]:
  209. ...
  210. @overload
  211. def get_global_option(
  212. checker: BaseChecker,
  213. option: GLOBAL_OPTION_PATTERN_LIST,
  214. default: list[Pattern[str]] | None = ...,
  215. ) -> list[Pattern[str]]:
  216. ...
  217. @overload
  218. def get_global_option(
  219. checker: BaseChecker,
  220. option: GLOBAL_OPTION_TUPLE_INT,
  221. default: tuple[int, ...] | None = ...,
  222. ) -> tuple[int, ...]:
  223. ...
  224. def get_global_option(
  225. checker: BaseChecker,
  226. option: GLOBAL_OPTION_NAMES,
  227. default: T_GlobalOptionReturnTypes | None = None, # pylint: disable=unused-argument
  228. ) -> T_GlobalOptionReturnTypes | None | Any:
  229. """DEPRECATED: Retrieve an option defined by the given *checker* or
  230. by all known option providers.
  231. It will look in the list of all options providers
  232. until the given *option* will be found.
  233. If the option wasn't found, the *default* value will be returned.
  234. """
  235. warnings.warn(
  236. "get_global_option has been deprecated. You can use "
  237. "checker.linter.config to get all global options instead.",
  238. DeprecationWarning,
  239. stacklevel=2,
  240. )
  241. return getattr(checker.linter.config, option.replace("-", "_"))
  242. def _splitstrip(string: str, sep: str = ",") -> list[str]:
  243. """Return a list of stripped string by splitting the string given as
  244. argument on `sep` (',' by default), empty strings are discarded.
  245. >>> _splitstrip('a, b, c , 4,,')
  246. ['a', 'b', 'c', '4']
  247. >>> _splitstrip('a')
  248. ['a']
  249. >>> _splitstrip('a,\nb,\nc,')
  250. ['a', 'b', 'c']
  251. :type string: str or unicode
  252. :param string: a csv line
  253. :type sep: str or unicode
  254. :param sep: field separator, default to the comma (',')
  255. :rtype: str or unicode
  256. :return: the unquoted string (or the input string if it wasn't quoted)
  257. """
  258. return [word.strip() for word in string.split(sep) if word.strip()]
  259. def _unquote(string: str) -> str:
  260. """Remove optional quotes (simple or double) from the string.
  261. :param string: an optionally quoted string
  262. :return: the unquoted string (or the input string if it wasn't quoted)
  263. """
  264. if not string:
  265. return string
  266. if string[0] in "\"'":
  267. string = string[1:]
  268. if string[-1] in "\"'":
  269. string = string[:-1]
  270. return string
  271. def _check_csv(value: list[str] | tuple[str] | str) -> Sequence[str]:
  272. if isinstance(value, (list, tuple)):
  273. return value
  274. return _splitstrip(value)
  275. def _check_regexp_csv(value: list[str] | tuple[str] | str) -> Iterable[str]:
  276. r"""Split a comma-separated list of regexps, taking care to avoid splitting
  277. a regex employing a comma as quantifier, as in `\d{1,2}`."""
  278. if isinstance(value, (list, tuple)):
  279. yield from value
  280. else:
  281. # None is a sentinel value here
  282. regexps: deque[deque[str] | None] = deque([None])
  283. open_braces = False
  284. for char in value:
  285. if char == "{":
  286. open_braces = True
  287. elif char == "}" and open_braces:
  288. open_braces = False
  289. if char == "," and not open_braces:
  290. regexps.append(None)
  291. elif regexps[-1] is None:
  292. regexps.pop()
  293. regexps.append(deque([char]))
  294. else:
  295. regexps[-1].append(char)
  296. yield from ("".join(regexp).strip() for regexp in regexps if regexp is not None)
  297. def _comment(string: str) -> str:
  298. """Return string as a comment."""
  299. lines = [line.strip() for line in string.splitlines()]
  300. sep = "\n"
  301. return "# " + f"{sep}# ".join(lines)
  302. def _format_option_value(optdict: OptionDict, value: Any) -> str:
  303. """Return the user input's value from a 'compiled' value.
  304. TODO: 3.0: Remove deprecated function
  305. """
  306. if optdict.get("type", None) == "py_version":
  307. value = ".".join(str(item) for item in value)
  308. elif isinstance(value, (list, tuple)):
  309. value = ",".join(_format_option_value(optdict, item) for item in value)
  310. elif isinstance(value, dict):
  311. value = ",".join(f"{k}:{v}" for k, v in value.items())
  312. elif hasattr(value, "match"): # optdict.get('type') == 'regexp'
  313. # compiled regexp
  314. value = value.pattern
  315. elif optdict.get("type") == "yn":
  316. value = "yes" if value else "no"
  317. elif isinstance(value, str) and value.isspace():
  318. value = f"'{value}'"
  319. return str(value)
  320. def format_section(
  321. stream: TextIO,
  322. section: str,
  323. options: list[tuple[str, OptionDict, Any]],
  324. doc: str | None = None,
  325. ) -> None:
  326. """Format an option's section using the INI format."""
  327. warnings.warn(
  328. "format_section has been deprecated. It will be removed in pylint 3.0.",
  329. DeprecationWarning,
  330. stacklevel=2,
  331. )
  332. if doc:
  333. print(_comment(doc), file=stream)
  334. print(f"[{section}]", file=stream)
  335. with warnings.catch_warnings():
  336. warnings.filterwarnings("ignore", category=DeprecationWarning)
  337. _ini_format(stream, options)
  338. def _ini_format(stream: TextIO, options: list[tuple[str, OptionDict, Any]]) -> None:
  339. """Format options using the INI format."""
  340. warnings.warn(
  341. "_ini_format has been deprecated. It will be removed in pylint 3.0.",
  342. DeprecationWarning,
  343. stacklevel=2,
  344. )
  345. for optname, optdict, value in options:
  346. # Skip deprecated option
  347. if "kwargs" in optdict:
  348. assert isinstance(optdict["kwargs"], dict)
  349. if "new_names" in optdict["kwargs"]:
  350. continue
  351. value = _format_option_value(optdict, value)
  352. help_opt = optdict.get("help")
  353. if help_opt:
  354. assert isinstance(help_opt, str)
  355. help_opt = normalize_text(help_opt, indent="# ")
  356. print(file=stream)
  357. print(help_opt, file=stream)
  358. else:
  359. print(file=stream)
  360. if value in {"None", "False"}:
  361. print(f"#{optname}=", file=stream)
  362. else:
  363. value = str(value).strip()
  364. if re.match(r"^([\w-]+,)+[\w-]+$", str(value)):
  365. separator = "\n " + " " * len(optname)
  366. value = separator.join(x + "," for x in str(value).split(","))
  367. # remove trailing ',' from last element of the list
  368. value = value[:-1]
  369. print(f"{optname}={value}", file=stream)
  370. class IsortDriver:
  371. """A wrapper around isort API that changed between versions 4 and 5."""
  372. def __init__(self, config: argparse.Namespace) -> None:
  373. if HAS_ISORT_5:
  374. self.isort5_config = isort.settings.Config(
  375. # There is no typo here. EXTRA_standard_library is
  376. # what most users want. The option has been named
  377. # KNOWN_standard_library for ages in pylint, and we
  378. # don't want to break compatibility.
  379. extra_standard_library=config.known_standard_library,
  380. known_third_party=config.known_third_party,
  381. )
  382. else:
  383. # pylint: disable-next=no-member
  384. self.isort4_obj = isort.SortImports( # type: ignore[attr-defined]
  385. file_contents="",
  386. known_standard_library=config.known_standard_library,
  387. known_third_party=config.known_third_party,
  388. )
  389. def place_module(self, package: str) -> str:
  390. if HAS_ISORT_5:
  391. return isort.api.place_module(package, self.isort5_config)
  392. return self.isort4_obj.place_module(package) # type: ignore[no-any-return]