utils.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. """Utility methods for flake8."""
  2. from __future__ import annotations
  3. import fnmatch as _fnmatch
  4. import functools
  5. import io
  6. import logging
  7. import os
  8. import platform
  9. import re
  10. import sys
  11. import textwrap
  12. import tokenize
  13. from typing import NamedTuple
  14. from typing import Pattern
  15. from typing import Sequence
  16. from flake8 import exceptions
  17. COMMA_SEPARATED_LIST_RE = re.compile(r"[,\s]")
  18. LOCAL_PLUGIN_LIST_RE = re.compile(r"[,\t\n\r\f\v]")
  19. NORMALIZE_PACKAGE_NAME_RE = re.compile(r"[-_.]+")
  20. def parse_comma_separated_list(
  21. value: str, regexp: Pattern[str] = COMMA_SEPARATED_LIST_RE
  22. ) -> list[str]:
  23. """Parse a comma-separated list.
  24. :param value:
  25. String to be parsed and normalized.
  26. :param regexp:
  27. Compiled regular expression used to split the value when it is a
  28. string.
  29. :returns:
  30. List of values with whitespace stripped.
  31. """
  32. assert isinstance(value, str), value
  33. separated = regexp.split(value)
  34. item_gen = (item.strip() for item in separated)
  35. return [item for item in item_gen if item]
  36. class _Token(NamedTuple):
  37. tp: str
  38. src: str
  39. _CODE, _FILE, _COLON, _COMMA, _WS = "code", "file", "colon", "comma", "ws"
  40. _EOF = "eof"
  41. _FILE_LIST_TOKEN_TYPES = [
  42. (re.compile(r"[A-Z]+[0-9]*(?=$|\s|,)"), _CODE),
  43. (re.compile(r"[^\s:,]+"), _FILE),
  44. (re.compile(r"\s*:\s*"), _COLON),
  45. (re.compile(r"\s*,\s*"), _COMMA),
  46. (re.compile(r"\s+"), _WS),
  47. ]
  48. def _tokenize_files_to_codes_mapping(value: str) -> list[_Token]:
  49. tokens = []
  50. i = 0
  51. while i < len(value):
  52. for token_re, token_name in _FILE_LIST_TOKEN_TYPES:
  53. match = token_re.match(value, i)
  54. if match:
  55. tokens.append(_Token(token_name, match.group().strip()))
  56. i = match.end()
  57. break
  58. else:
  59. raise AssertionError("unreachable", value, i)
  60. tokens.append(_Token(_EOF, ""))
  61. return tokens
  62. def parse_files_to_codes_mapping( # noqa: C901
  63. value_: Sequence[str] | str,
  64. ) -> list[tuple[str, list[str]]]:
  65. """Parse a files-to-codes mapping.
  66. A files-to-codes mapping a sequence of values specified as
  67. `filenames list:codes list ...`. Each of the lists may be separated by
  68. either comma or whitespace tokens.
  69. :param value: String to be parsed and normalized.
  70. """
  71. if not isinstance(value_, str):
  72. value = "\n".join(value_)
  73. else:
  74. value = value_
  75. ret: list[tuple[str, list[str]]] = []
  76. if not value.strip():
  77. return ret
  78. class State:
  79. seen_sep = True
  80. seen_colon = False
  81. filenames: list[str] = []
  82. codes: list[str] = []
  83. def _reset() -> None:
  84. if State.codes:
  85. for filename in State.filenames:
  86. ret.append((filename, State.codes))
  87. State.seen_sep = True
  88. State.seen_colon = False
  89. State.filenames = []
  90. State.codes = []
  91. def _unexpected_token() -> exceptions.ExecutionError:
  92. return exceptions.ExecutionError(
  93. f"Expected `per-file-ignores` to be a mapping from file exclude "
  94. f"patterns to ignore codes.\n\n"
  95. f"Configured `per-file-ignores` setting:\n\n"
  96. f"{textwrap.indent(value.strip(), ' ')}"
  97. )
  98. for token in _tokenize_files_to_codes_mapping(value):
  99. # legal in any state: separator sets the sep bit
  100. if token.tp in {_COMMA, _WS}:
  101. State.seen_sep = True
  102. # looking for filenames
  103. elif not State.seen_colon:
  104. if token.tp == _COLON:
  105. State.seen_colon = True
  106. State.seen_sep = True
  107. elif State.seen_sep and token.tp == _FILE:
  108. State.filenames.append(token.src)
  109. State.seen_sep = False
  110. else:
  111. raise _unexpected_token()
  112. # looking for codes
  113. else:
  114. if token.tp == _EOF:
  115. _reset()
  116. elif State.seen_sep and token.tp == _CODE:
  117. State.codes.append(token.src)
  118. State.seen_sep = False
  119. elif State.seen_sep and token.tp == _FILE:
  120. _reset()
  121. State.filenames.append(token.src)
  122. State.seen_sep = False
  123. else:
  124. raise _unexpected_token()
  125. return ret
  126. def normalize_paths(
  127. paths: Sequence[str], parent: str = os.curdir
  128. ) -> list[str]:
  129. """Normalize a list of paths relative to a parent directory.
  130. :returns:
  131. The normalized paths.
  132. """
  133. assert isinstance(paths, list), paths
  134. return [normalize_path(p, parent) for p in paths]
  135. def normalize_path(path: str, parent: str = os.curdir) -> str:
  136. """Normalize a single-path.
  137. :returns:
  138. The normalized path.
  139. """
  140. # NOTE(sigmavirus24): Using os.path.sep and os.path.altsep allow for
  141. # Windows compatibility with both Windows-style paths (c:\foo\bar) and
  142. # Unix style paths (/foo/bar).
  143. separator = os.path.sep
  144. # NOTE(sigmavirus24): os.path.altsep may be None
  145. alternate_separator = os.path.altsep or ""
  146. if (
  147. path == "."
  148. or separator in path
  149. or (alternate_separator and alternate_separator in path)
  150. ):
  151. path = os.path.abspath(os.path.join(parent, path))
  152. return path.rstrip(separator + alternate_separator)
  153. @functools.lru_cache(maxsize=1)
  154. def stdin_get_value() -> str:
  155. """Get and cache it so plugins can use it."""
  156. stdin_value = sys.stdin.buffer.read()
  157. fd = io.BytesIO(stdin_value)
  158. try:
  159. coding, _ = tokenize.detect_encoding(fd.readline)
  160. fd.seek(0)
  161. return io.TextIOWrapper(fd, coding).read()
  162. except (LookupError, SyntaxError, UnicodeError):
  163. return stdin_value.decode("utf-8")
  164. def stdin_get_lines() -> list[str]:
  165. """Return lines of stdin split according to file splitting."""
  166. return list(io.StringIO(stdin_get_value()))
  167. def is_using_stdin(paths: list[str]) -> bool:
  168. """Determine if we're going to read from stdin.
  169. :param paths:
  170. The paths that we're going to check.
  171. :returns:
  172. True if stdin (-) is in the path, otherwise False
  173. """
  174. return "-" in paths
  175. def fnmatch(filename: str, patterns: Sequence[str]) -> bool:
  176. """Wrap :func:`fnmatch.fnmatch` to add some functionality.
  177. :param filename:
  178. Name of the file we're trying to match.
  179. :param patterns:
  180. Patterns we're using to try to match the filename.
  181. :param default:
  182. The default value if patterns is empty
  183. :returns:
  184. True if a pattern matches the filename, False if it doesn't.
  185. ``True`` if patterns is empty.
  186. """
  187. if not patterns:
  188. return True
  189. return any(_fnmatch.fnmatch(filename, pattern) for pattern in patterns)
  190. def matches_filename(
  191. path: str,
  192. patterns: Sequence[str],
  193. log_message: str,
  194. logger: logging.Logger,
  195. ) -> bool:
  196. """Use fnmatch to discern if a path exists in patterns.
  197. :param path:
  198. The path to the file under question
  199. :param patterns:
  200. The patterns to match the path against.
  201. :param log_message:
  202. The message used for logging purposes.
  203. :returns:
  204. True if path matches patterns, False otherwise
  205. """
  206. if not patterns:
  207. return False
  208. basename = os.path.basename(path)
  209. if basename not in {".", ".."} and fnmatch(basename, patterns):
  210. logger.debug(log_message, {"path": basename, "whether": ""})
  211. return True
  212. absolute_path = os.path.abspath(path)
  213. match = fnmatch(absolute_path, patterns)
  214. logger.debug(
  215. log_message,
  216. {"path": absolute_path, "whether": "" if match else "not "},
  217. )
  218. return match
  219. def get_python_version() -> str:
  220. """Find and format the python implementation and version.
  221. :returns:
  222. Implementation name, version, and platform as a string.
  223. """
  224. return "{} {} on {}".format(
  225. platform.python_implementation(),
  226. platform.python_version(),
  227. platform.system(),
  228. )
  229. def normalize_pypi_name(s: str) -> str:
  230. """Normalize a distribution name according to PEP 503."""
  231. return NORMALIZE_PACKAGE_NAME_RE.sub("-", s).lower()