fastparse.py 79 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100
  1. from __future__ import annotations
  2. import copy
  3. import re
  4. import sys
  5. import warnings
  6. from typing import Any, Callable, Final, List, Optional, Sequence, TypeVar, Union, cast
  7. from typing_extensions import Literal, overload
  8. from mypy import defaults, errorcodes as codes, message_registry
  9. from mypy.errors import Errors
  10. from mypy.message_registry import ErrorMessage
  11. from mypy.nodes import (
  12. ARG_NAMED,
  13. ARG_NAMED_OPT,
  14. ARG_OPT,
  15. ARG_POS,
  16. ARG_STAR,
  17. ARG_STAR2,
  18. ArgKind,
  19. Argument,
  20. AssertStmt,
  21. AssignmentExpr,
  22. AssignmentStmt,
  23. AwaitExpr,
  24. Block,
  25. BreakStmt,
  26. BytesExpr,
  27. CallExpr,
  28. ClassDef,
  29. ComparisonExpr,
  30. ComplexExpr,
  31. ConditionalExpr,
  32. ContinueStmt,
  33. Decorator,
  34. DelStmt,
  35. DictExpr,
  36. DictionaryComprehension,
  37. EllipsisExpr,
  38. Expression,
  39. ExpressionStmt,
  40. FakeInfo,
  41. FloatExpr,
  42. ForStmt,
  43. FuncDef,
  44. GeneratorExpr,
  45. GlobalDecl,
  46. IfStmt,
  47. Import,
  48. ImportAll,
  49. ImportBase,
  50. ImportFrom,
  51. IndexExpr,
  52. IntExpr,
  53. LambdaExpr,
  54. ListComprehension,
  55. ListExpr,
  56. MatchStmt,
  57. MemberExpr,
  58. MypyFile,
  59. NameExpr,
  60. Node,
  61. NonlocalDecl,
  62. OperatorAssignmentStmt,
  63. OpExpr,
  64. OverloadedFuncDef,
  65. OverloadPart,
  66. PassStmt,
  67. RaiseStmt,
  68. RefExpr,
  69. ReturnStmt,
  70. SetComprehension,
  71. SetExpr,
  72. SliceExpr,
  73. StarExpr,
  74. Statement,
  75. StrExpr,
  76. SuperExpr,
  77. TempNode,
  78. TryStmt,
  79. TupleExpr,
  80. UnaryExpr,
  81. Var,
  82. WhileStmt,
  83. WithStmt,
  84. YieldExpr,
  85. YieldFromExpr,
  86. check_arg_names,
  87. )
  88. from mypy.options import Options
  89. from mypy.patterns import (
  90. AsPattern,
  91. ClassPattern,
  92. MappingPattern,
  93. OrPattern,
  94. SequencePattern,
  95. SingletonPattern,
  96. StarredPattern,
  97. ValuePattern,
  98. )
  99. from mypy.reachability import infer_reachability_of_if_statement, mark_block_unreachable
  100. from mypy.sharedparse import argument_elide_name, special_function_elide_names
  101. from mypy.traverser import TraverserVisitor
  102. from mypy.types import (
  103. AnyType,
  104. CallableArgument,
  105. CallableType,
  106. EllipsisType,
  107. Instance,
  108. ProperType,
  109. RawExpressionType,
  110. TupleType,
  111. Type,
  112. TypeList,
  113. TypeOfAny,
  114. UnboundType,
  115. UnionType,
  116. )
  117. from mypy.util import bytes_to_human_readable_repr, unnamed_function
  118. # pull this into a final variable to make mypyc be quiet about the
  119. # the default argument warning
  120. PY_MINOR_VERSION: Final = sys.version_info[1]
  121. import ast as ast3
  122. # TODO: Index, ExtSlice are deprecated in 3.9.
  123. from ast import AST, Attribute, Call, FunctionType, Index, Name, Starred, UnaryOp, USub
  124. def ast3_parse(
  125. source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION
  126. ) -> AST:
  127. return ast3.parse(
  128. source,
  129. filename,
  130. mode,
  131. type_comments=True, # This works the magic
  132. feature_version=feature_version,
  133. )
  134. NamedExpr = ast3.NamedExpr
  135. Constant = ast3.Constant
  136. if sys.version_info >= (3, 10):
  137. Match = ast3.Match
  138. MatchValue = ast3.MatchValue
  139. MatchSingleton = ast3.MatchSingleton
  140. MatchSequence = ast3.MatchSequence
  141. MatchStar = ast3.MatchStar
  142. MatchMapping = ast3.MatchMapping
  143. MatchClass = ast3.MatchClass
  144. MatchAs = ast3.MatchAs
  145. MatchOr = ast3.MatchOr
  146. AstNode = Union[ast3.expr, ast3.stmt, ast3.pattern, ast3.ExceptHandler]
  147. else:
  148. Match = Any
  149. MatchValue = Any
  150. MatchSingleton = Any
  151. MatchSequence = Any
  152. MatchStar = Any
  153. MatchMapping = Any
  154. MatchClass = Any
  155. MatchAs = Any
  156. MatchOr = Any
  157. AstNode = Union[ast3.expr, ast3.stmt, ast3.ExceptHandler]
  158. if sys.version_info >= (3, 11):
  159. TryStar = ast3.TryStar
  160. else:
  161. TryStar = Any
  162. N = TypeVar("N", bound=Node)
  163. # There is no way to create reasonable fallbacks at this stage,
  164. # they must be patched later.
  165. MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal")
  166. _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1)
  167. TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)")
  168. def parse(
  169. source: str | bytes,
  170. fnam: str,
  171. module: str | None,
  172. errors: Errors | None = None,
  173. options: Options | None = None,
  174. ) -> MypyFile:
  175. """Parse a source file, without doing any semantic analysis.
  176. Return the parse tree. If errors is not provided, raise ParseError
  177. on failure. Otherwise, use the errors object to report parse errors.
  178. """
  179. ignore_errors = (options is not None and options.ignore_errors) or (
  180. errors is not None and fnam in errors.ignored_files
  181. )
  182. # If errors are ignored, we can drop many function bodies to speed up type checking.
  183. strip_function_bodies = ignore_errors and (options is None or not options.preserve_asts)
  184. raise_on_error = False
  185. if options is None:
  186. options = Options()
  187. if errors is None:
  188. errors = Errors(options)
  189. raise_on_error = True
  190. errors.set_file(fnam, module, options=options)
  191. is_stub_file = fnam.endswith(".pyi")
  192. if is_stub_file:
  193. feature_version = defaults.PYTHON3_VERSION[1]
  194. if options.python_version[0] == 3 and options.python_version[1] > feature_version:
  195. feature_version = options.python_version[1]
  196. else:
  197. assert options.python_version[0] >= 3
  198. feature_version = options.python_version[1]
  199. try:
  200. # Disable deprecation warnings about \u
  201. with warnings.catch_warnings():
  202. warnings.filterwarnings("ignore", category=DeprecationWarning)
  203. ast = ast3_parse(source, fnam, "exec", feature_version=feature_version)
  204. tree = ASTConverter(
  205. options=options,
  206. is_stub=is_stub_file,
  207. errors=errors,
  208. ignore_errors=ignore_errors,
  209. strip_function_bodies=strip_function_bodies,
  210. ).visit(ast)
  211. tree.path = fnam
  212. tree.is_stub = is_stub_file
  213. except SyntaxError as e:
  214. # alias to please mypyc
  215. is_py38_or_earlier = sys.version_info < (3, 9)
  216. if is_py38_or_earlier and e.filename == "<fstring>":
  217. # In Python 3.8 and earlier, syntax errors in f-strings have lineno relative to the
  218. # start of the f-string. This would be misleading, as mypy will report the error as the
  219. # lineno within the file.
  220. e.lineno = None
  221. message = e.msg
  222. if feature_version > sys.version_info.minor and message.startswith("invalid syntax"):
  223. python_version_str = f"{options.python_version[0]}.{options.python_version[1]}"
  224. message += f"; you likely need to run mypy using Python {python_version_str} or newer"
  225. errors.report(
  226. e.lineno if e.lineno is not None else -1,
  227. e.offset,
  228. message,
  229. blocker=True,
  230. code=codes.SYNTAX,
  231. )
  232. tree = MypyFile([], [], False, {})
  233. if raise_on_error and errors.is_errors():
  234. errors.raise_error()
  235. assert isinstance(tree, MypyFile)
  236. return tree
  237. def parse_type_ignore_tag(tag: str | None) -> list[str] | None:
  238. """Parse optional "[code, ...]" tag after "# type: ignore".
  239. Return:
  240. * [] if no tag was found (ignore all errors)
  241. * list of ignored error codes if a tag was found
  242. * None if the tag was invalid.
  243. """
  244. if not tag or tag.strip() == "" or tag.strip().startswith("#"):
  245. # No tag -- ignore all errors.
  246. return []
  247. m = re.match(r"\s*\[([^]#]*)\]\s*(#.*)?$", tag)
  248. if m is None:
  249. # Invalid "# type: ignore" comment.
  250. return None
  251. return [code.strip() for code in m.group(1).split(",")]
  252. def parse_type_comment(
  253. type_comment: str, line: int, column: int, errors: Errors | None
  254. ) -> tuple[list[str] | None, ProperType | None]:
  255. """Parse type portion of a type comment (+ optional type ignore).
  256. Return (ignore info, parsed type).
  257. """
  258. try:
  259. typ = ast3_parse(type_comment, "<type_comment>", "eval")
  260. except SyntaxError:
  261. if errors is not None:
  262. stripped_type = type_comment.split("#", 2)[0].strip()
  263. err_msg = message_registry.TYPE_COMMENT_SYNTAX_ERROR_VALUE.format(stripped_type)
  264. errors.report(line, column, err_msg.value, blocker=True, code=err_msg.code)
  265. return None, None
  266. else:
  267. raise
  268. else:
  269. extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment)
  270. if extra_ignore:
  271. tag: str | None = extra_ignore.group(1)
  272. ignored: list[str] | None = parse_type_ignore_tag(tag)
  273. if ignored is None:
  274. if errors is not None:
  275. errors.report(
  276. line, column, message_registry.INVALID_TYPE_IGNORE.value, code=codes.SYNTAX
  277. )
  278. else:
  279. raise SyntaxError
  280. else:
  281. ignored = None
  282. assert isinstance(typ, ast3.Expression)
  283. converted = TypeConverter(
  284. errors, line=line, override_column=column, is_evaluated=False
  285. ).visit(typ.body)
  286. return ignored, converted
  287. def parse_type_string(
  288. expr_string: str, expr_fallback_name: str, line: int, column: int
  289. ) -> ProperType:
  290. """Parses a type that was originally present inside of an explicit string.
  291. For example, suppose we have the type `Foo["blah"]`. We should parse the
  292. string expression "blah" using this function.
  293. """
  294. try:
  295. _, node = parse_type_comment(expr_string.strip(), line=line, column=column, errors=None)
  296. if isinstance(node, UnboundType) and node.original_str_expr is None:
  297. node.original_str_expr = expr_string
  298. node.original_str_fallback = expr_fallback_name
  299. return node
  300. elif isinstance(node, UnionType):
  301. return node
  302. else:
  303. return RawExpressionType(expr_string, expr_fallback_name, line, column)
  304. except (SyntaxError, ValueError):
  305. # Note: the parser will raise a `ValueError` instead of a SyntaxError if
  306. # the string happens to contain things like \x00.
  307. return RawExpressionType(expr_string, expr_fallback_name, line, column)
  308. def is_no_type_check_decorator(expr: ast3.expr) -> bool:
  309. if isinstance(expr, Name):
  310. return expr.id == "no_type_check"
  311. elif isinstance(expr, Attribute):
  312. if isinstance(expr.value, Name):
  313. return expr.value.id == "typing" and expr.attr == "no_type_check"
  314. return False
  315. class ASTConverter:
  316. def __init__(
  317. self,
  318. options: Options,
  319. is_stub: bool,
  320. errors: Errors,
  321. *,
  322. ignore_errors: bool,
  323. strip_function_bodies: bool,
  324. ) -> None:
  325. # 'C' for class, 'D' for function signature, 'F' for function, 'L' for lambda
  326. self.class_and_function_stack: list[Literal["C", "D", "F", "L"]] = []
  327. self.imports: list[ImportBase] = []
  328. self.options = options
  329. self.is_stub = is_stub
  330. self.errors = errors
  331. self.ignore_errors = ignore_errors
  332. self.strip_function_bodies = strip_function_bodies
  333. self.type_ignores: dict[int, list[str]] = {}
  334. # Cache of visit_X methods keyed by type of visited object
  335. self.visitor_cache: dict[type, Callable[[AST | None], Any]] = {}
  336. def note(self, msg: str, line: int, column: int) -> None:
  337. self.errors.report(line, column, msg, severity="note", code=codes.SYNTAX)
  338. def fail(self, msg: ErrorMessage, line: int, column: int, blocker: bool = True) -> None:
  339. if blocker or not self.options.ignore_errors:
  340. self.errors.report(line, column, msg.value, blocker=blocker, code=msg.code)
  341. def fail_merge_overload(self, node: IfStmt) -> None:
  342. self.fail(
  343. message_registry.FAILED_TO_MERGE_OVERLOADS,
  344. line=node.line,
  345. column=node.column,
  346. blocker=False,
  347. )
  348. def visit(self, node: AST | None) -> Any:
  349. if node is None:
  350. return None
  351. typeobj = type(node)
  352. visitor = self.visitor_cache.get(typeobj)
  353. if visitor is None:
  354. method = "visit_" + node.__class__.__name__
  355. visitor = getattr(self, method)
  356. self.visitor_cache[typeobj] = visitor
  357. return visitor(node)
  358. def set_line(self, node: N, n: AstNode) -> N:
  359. node.line = n.lineno
  360. node.column = n.col_offset
  361. node.end_line = getattr(n, "end_lineno", None)
  362. node.end_column = getattr(n, "end_col_offset", None)
  363. return node
  364. def translate_opt_expr_list(self, l: Sequence[AST | None]) -> list[Expression | None]:
  365. res: list[Expression | None] = []
  366. for e in l:
  367. exp = self.visit(e)
  368. res.append(exp)
  369. return res
  370. def translate_expr_list(self, l: Sequence[AST]) -> list[Expression]:
  371. return cast(List[Expression], self.translate_opt_expr_list(l))
  372. def get_lineno(self, node: ast3.expr | ast3.stmt) -> int:
  373. if (
  374. isinstance(node, (ast3.AsyncFunctionDef, ast3.ClassDef, ast3.FunctionDef))
  375. and node.decorator_list
  376. ):
  377. return node.decorator_list[0].lineno
  378. return node.lineno
  379. def translate_stmt_list(
  380. self,
  381. stmts: Sequence[ast3.stmt],
  382. *,
  383. ismodule: bool = False,
  384. can_strip: bool = False,
  385. is_coroutine: bool = False,
  386. ) -> list[Statement]:
  387. # A "# type: ignore" comment before the first statement of a module
  388. # ignores the whole module:
  389. if (
  390. ismodule
  391. and stmts
  392. and self.type_ignores
  393. and min(self.type_ignores) < self.get_lineno(stmts[0])
  394. ):
  395. ignores = self.type_ignores[min(self.type_ignores)]
  396. if ignores:
  397. joined_ignores = ", ".join(ignores)
  398. self.fail(
  399. message_registry.TYPE_IGNORE_WITH_ERRCODE_ON_MODULE.format(joined_ignores),
  400. line=min(self.type_ignores),
  401. column=0,
  402. blocker=False,
  403. )
  404. self.errors.used_ignored_lines[self.errors.file][min(self.type_ignores)].append(
  405. codes.FILE.code
  406. )
  407. block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts)))
  408. self.set_block_lines(block, stmts)
  409. mark_block_unreachable(block)
  410. return [block]
  411. stack = self.class_and_function_stack
  412. # Fast case for stripping function bodies
  413. if (
  414. can_strip
  415. and self.strip_function_bodies
  416. and len(stack) == 1
  417. and stack[0] == "F"
  418. and not is_coroutine
  419. ):
  420. return []
  421. res: list[Statement] = []
  422. for stmt in stmts:
  423. node = self.visit(stmt)
  424. res.append(node)
  425. # Slow case for stripping function bodies
  426. if can_strip and self.strip_function_bodies:
  427. if stack[-2:] == ["C", "F"]:
  428. if is_possible_trivial_body(res):
  429. can_strip = False
  430. else:
  431. # We only strip method bodies if they don't assign to an attribute, as
  432. # this may define an attribute which has an externally visible effect.
  433. visitor = FindAttributeAssign()
  434. for s in res:
  435. s.accept(visitor)
  436. if visitor.found:
  437. can_strip = False
  438. break
  439. if can_strip and stack[-1] == "F" and is_coroutine:
  440. # Yields inside an async function affect the return type and should not
  441. # be stripped.
  442. yield_visitor = FindYield()
  443. for s in res:
  444. s.accept(yield_visitor)
  445. if yield_visitor.found:
  446. can_strip = False
  447. break
  448. if can_strip:
  449. return []
  450. return res
  451. def translate_type_comment(
  452. self, n: ast3.stmt | ast3.arg, type_comment: str | None
  453. ) -> ProperType | None:
  454. if type_comment is None:
  455. return None
  456. else:
  457. lineno = n.lineno
  458. extra_ignore, typ = parse_type_comment(type_comment, lineno, n.col_offset, self.errors)
  459. if extra_ignore is not None:
  460. self.type_ignores[lineno] = extra_ignore
  461. return typ
  462. op_map: Final[dict[type[AST], str]] = {
  463. ast3.Add: "+",
  464. ast3.Sub: "-",
  465. ast3.Mult: "*",
  466. ast3.MatMult: "@",
  467. ast3.Div: "/",
  468. ast3.Mod: "%",
  469. ast3.Pow: "**",
  470. ast3.LShift: "<<",
  471. ast3.RShift: ">>",
  472. ast3.BitOr: "|",
  473. ast3.BitXor: "^",
  474. ast3.BitAnd: "&",
  475. ast3.FloorDiv: "//",
  476. }
  477. def from_operator(self, op: ast3.operator) -> str:
  478. op_name = ASTConverter.op_map.get(type(op))
  479. if op_name is None:
  480. raise RuntimeError("Unknown operator " + str(type(op)))
  481. else:
  482. return op_name
  483. comp_op_map: Final[dict[type[AST], str]] = {
  484. ast3.Gt: ">",
  485. ast3.Lt: "<",
  486. ast3.Eq: "==",
  487. ast3.GtE: ">=",
  488. ast3.LtE: "<=",
  489. ast3.NotEq: "!=",
  490. ast3.Is: "is",
  491. ast3.IsNot: "is not",
  492. ast3.In: "in",
  493. ast3.NotIn: "not in",
  494. }
  495. def from_comp_operator(self, op: ast3.cmpop) -> str:
  496. op_name = ASTConverter.comp_op_map.get(type(op))
  497. if op_name is None:
  498. raise RuntimeError("Unknown comparison operator " + str(type(op)))
  499. else:
  500. return op_name
  501. def set_block_lines(self, b: Block, stmts: Sequence[ast3.stmt]) -> None:
  502. first, last = stmts[0], stmts[-1]
  503. b.line = first.lineno
  504. b.column = first.col_offset
  505. b.end_line = getattr(last, "end_lineno", None)
  506. b.end_column = getattr(last, "end_col_offset", None)
  507. if not b.body:
  508. return
  509. new_first = b.body[0]
  510. if isinstance(new_first, (Decorator, OverloadedFuncDef)):
  511. # Decorated function lines are different between Python versions.
  512. # copy the normalization we do for them to block first lines.
  513. b.line = new_first.line
  514. b.column = new_first.column
  515. def as_block(self, stmts: list[ast3.stmt]) -> Block | None:
  516. b = None
  517. if stmts:
  518. b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts)))
  519. self.set_block_lines(b, stmts)
  520. return b
  521. def as_required_block(
  522. self, stmts: list[ast3.stmt], *, can_strip: bool = False, is_coroutine: bool = False
  523. ) -> Block:
  524. assert stmts # must be non-empty
  525. b = Block(
  526. self.fix_function_overloads(
  527. self.translate_stmt_list(stmts, can_strip=can_strip, is_coroutine=is_coroutine)
  528. )
  529. )
  530. self.set_block_lines(b, stmts)
  531. return b
  532. def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]:
  533. ret: list[Statement] = []
  534. current_overload: list[OverloadPart] = []
  535. current_overload_name: str | None = None
  536. seen_unconditional_func_def = False
  537. last_if_stmt: IfStmt | None = None
  538. last_if_overload: Decorator | FuncDef | OverloadedFuncDef | None = None
  539. last_if_stmt_overload_name: str | None = None
  540. last_if_unknown_truth_value: IfStmt | None = None
  541. skipped_if_stmts: list[IfStmt] = []
  542. for stmt in stmts:
  543. if_overload_name: str | None = None
  544. if_block_with_overload: Block | None = None
  545. if_unknown_truth_value: IfStmt | None = None
  546. if isinstance(stmt, IfStmt) and seen_unconditional_func_def is False:
  547. # Check IfStmt block to determine if function overloads can be merged
  548. if_overload_name = self._check_ifstmt_for_overloads(stmt, current_overload_name)
  549. if if_overload_name is not None:
  550. (
  551. if_block_with_overload,
  552. if_unknown_truth_value,
  553. ) = self._get_executable_if_block_with_overloads(stmt)
  554. if (
  555. current_overload_name is not None
  556. and isinstance(stmt, (Decorator, FuncDef))
  557. and stmt.name == current_overload_name
  558. ):
  559. if last_if_stmt is not None:
  560. skipped_if_stmts.append(last_if_stmt)
  561. if last_if_overload is not None:
  562. # Last stmt was an IfStmt with same overload name
  563. # Add overloads to current_overload
  564. if isinstance(last_if_overload, OverloadedFuncDef):
  565. current_overload.extend(last_if_overload.items)
  566. else:
  567. current_overload.append(last_if_overload)
  568. last_if_stmt, last_if_overload = None, None
  569. if last_if_unknown_truth_value:
  570. self.fail_merge_overload(last_if_unknown_truth_value)
  571. last_if_unknown_truth_value = None
  572. current_overload.append(stmt)
  573. if isinstance(stmt, FuncDef):
  574. seen_unconditional_func_def = True
  575. elif (
  576. current_overload_name is not None
  577. and isinstance(stmt, IfStmt)
  578. and if_overload_name == current_overload_name
  579. ):
  580. # IfStmt only contains stmts relevant to current_overload.
  581. # Check if stmts are reachable and add them to current_overload,
  582. # otherwise skip IfStmt to allow subsequent overload
  583. # or function definitions.
  584. skipped_if_stmts.append(stmt)
  585. if if_block_with_overload is None:
  586. if if_unknown_truth_value is not None:
  587. self.fail_merge_overload(if_unknown_truth_value)
  588. continue
  589. if last_if_overload is not None:
  590. # Last stmt was an IfStmt with same overload name
  591. # Add overloads to current_overload
  592. if isinstance(last_if_overload, OverloadedFuncDef):
  593. current_overload.extend(last_if_overload.items)
  594. else:
  595. current_overload.append(last_if_overload)
  596. last_if_stmt, last_if_overload = None, None
  597. if isinstance(if_block_with_overload.body[-1], OverloadedFuncDef):
  598. skipped_if_stmts.extend(cast(List[IfStmt], if_block_with_overload.body[:-1]))
  599. current_overload.extend(if_block_with_overload.body[-1].items)
  600. else:
  601. current_overload.append(
  602. cast(Union[Decorator, FuncDef], if_block_with_overload.body[0])
  603. )
  604. else:
  605. if last_if_stmt is not None:
  606. ret.append(last_if_stmt)
  607. last_if_stmt_overload_name = current_overload_name
  608. last_if_stmt, last_if_overload = None, None
  609. last_if_unknown_truth_value = None
  610. if current_overload and current_overload_name == last_if_stmt_overload_name:
  611. # Remove last stmt (IfStmt) from ret if the overload names matched
  612. # Only happens if no executable block had been found in IfStmt
  613. popped = ret.pop()
  614. assert isinstance(popped, IfStmt)
  615. skipped_if_stmts.append(popped)
  616. if current_overload and skipped_if_stmts:
  617. # Add bare IfStmt (without overloads) to ret
  618. # Required for mypy to be able to still check conditions
  619. for if_stmt in skipped_if_stmts:
  620. self._strip_contents_from_if_stmt(if_stmt)
  621. ret.append(if_stmt)
  622. skipped_if_stmts = []
  623. if len(current_overload) == 1:
  624. ret.append(current_overload[0])
  625. elif len(current_overload) > 1:
  626. ret.append(OverloadedFuncDef(current_overload))
  627. # If we have multiple decorated functions named "_" next to each, we want to treat
  628. # them as a series of regular FuncDefs instead of one OverloadedFuncDef because
  629. # most of mypy/mypyc assumes that all the functions in an OverloadedFuncDef are
  630. # related, but multiple underscore functions next to each other aren't necessarily
  631. # related
  632. seen_unconditional_func_def = False
  633. if isinstance(stmt, Decorator) and not unnamed_function(stmt.name):
  634. current_overload = [stmt]
  635. current_overload_name = stmt.name
  636. elif isinstance(stmt, IfStmt) and if_overload_name is not None:
  637. current_overload = []
  638. current_overload_name = if_overload_name
  639. last_if_stmt = stmt
  640. last_if_stmt_overload_name = None
  641. if if_block_with_overload is not None:
  642. skipped_if_stmts.extend(
  643. cast(List[IfStmt], if_block_with_overload.body[:-1])
  644. )
  645. last_if_overload = cast(
  646. Union[Decorator, FuncDef, OverloadedFuncDef],
  647. if_block_with_overload.body[-1],
  648. )
  649. last_if_unknown_truth_value = if_unknown_truth_value
  650. else:
  651. current_overload = []
  652. current_overload_name = None
  653. ret.append(stmt)
  654. if current_overload and skipped_if_stmts:
  655. # Add bare IfStmt (without overloads) to ret
  656. # Required for mypy to be able to still check conditions
  657. for if_stmt in skipped_if_stmts:
  658. self._strip_contents_from_if_stmt(if_stmt)
  659. ret.append(if_stmt)
  660. if len(current_overload) == 1:
  661. ret.append(current_overload[0])
  662. elif len(current_overload) > 1:
  663. ret.append(OverloadedFuncDef(current_overload))
  664. elif last_if_overload is not None:
  665. ret.append(last_if_overload)
  666. elif last_if_stmt is not None:
  667. ret.append(last_if_stmt)
  668. return ret
  669. def _check_ifstmt_for_overloads(
  670. self, stmt: IfStmt, current_overload_name: str | None = None
  671. ) -> str | None:
  672. """Check if IfStmt contains only overloads with the same name.
  673. Return overload_name if found, None otherwise.
  674. """
  675. # Check that block only contains a single Decorator, FuncDef, or OverloadedFuncDef.
  676. # Multiple overloads have already been merged as OverloadedFuncDef.
  677. if not (
  678. len(stmt.body[0].body) == 1
  679. and (
  680. isinstance(stmt.body[0].body[0], (Decorator, OverloadedFuncDef))
  681. or current_overload_name is not None
  682. and isinstance(stmt.body[0].body[0], FuncDef)
  683. )
  684. or len(stmt.body[0].body) > 1
  685. and isinstance(stmt.body[0].body[-1], OverloadedFuncDef)
  686. and all(self._is_stripped_if_stmt(if_stmt) for if_stmt in stmt.body[0].body[:-1])
  687. ):
  688. return None
  689. overload_name = cast(
  690. Union[Decorator, FuncDef, OverloadedFuncDef], stmt.body[0].body[-1]
  691. ).name
  692. if stmt.else_body is None:
  693. return overload_name
  694. if len(stmt.else_body.body) == 1:
  695. # For elif: else_body contains an IfStmt itself -> do a recursive check.
  696. if (
  697. isinstance(stmt.else_body.body[0], (Decorator, FuncDef, OverloadedFuncDef))
  698. and stmt.else_body.body[0].name == overload_name
  699. ):
  700. return overload_name
  701. if (
  702. isinstance(stmt.else_body.body[0], IfStmt)
  703. and self._check_ifstmt_for_overloads(stmt.else_body.body[0], current_overload_name)
  704. == overload_name
  705. ):
  706. return overload_name
  707. return None
  708. def _get_executable_if_block_with_overloads(
  709. self, stmt: IfStmt
  710. ) -> tuple[Block | None, IfStmt | None]:
  711. """Return block from IfStmt that will get executed.
  712. Return
  713. 0 -> A block if sure that alternative blocks are unreachable.
  714. 1 -> An IfStmt if the reachability of it can't be inferred,
  715. i.e. the truth value is unknown.
  716. """
  717. infer_reachability_of_if_statement(stmt, self.options)
  718. if stmt.else_body is None and stmt.body[0].is_unreachable is True:
  719. # always False condition with no else
  720. return None, None
  721. if (
  722. stmt.else_body is None
  723. or stmt.body[0].is_unreachable is False
  724. and stmt.else_body.is_unreachable is False
  725. ):
  726. # The truth value is unknown, thus not conclusive
  727. return None, stmt
  728. if stmt.else_body.is_unreachable is True:
  729. # else_body will be set unreachable if condition is always True
  730. return stmt.body[0], None
  731. if stmt.body[0].is_unreachable is True:
  732. # body will be set unreachable if condition is always False
  733. # else_body can contain an IfStmt itself (for elif) -> do a recursive check
  734. if isinstance(stmt.else_body.body[0], IfStmt):
  735. return self._get_executable_if_block_with_overloads(stmt.else_body.body[0])
  736. return stmt.else_body, None
  737. return None, stmt
  738. def _strip_contents_from_if_stmt(self, stmt: IfStmt) -> None:
  739. """Remove contents from IfStmt.
  740. Needed to still be able to check the conditions after the contents
  741. have been merged with the surrounding function overloads.
  742. """
  743. if len(stmt.body) == 1:
  744. stmt.body[0].body = []
  745. if stmt.else_body and len(stmt.else_body.body) == 1:
  746. if isinstance(stmt.else_body.body[0], IfStmt):
  747. self._strip_contents_from_if_stmt(stmt.else_body.body[0])
  748. else:
  749. stmt.else_body.body = []
  750. def _is_stripped_if_stmt(self, stmt: Statement) -> bool:
  751. """Check stmt to make sure it is a stripped IfStmt.
  752. See also: _strip_contents_from_if_stmt
  753. """
  754. if not isinstance(stmt, IfStmt):
  755. return False
  756. if not (len(stmt.body) == 1 and len(stmt.body[0].body) == 0):
  757. # Body not empty
  758. return False
  759. if not stmt.else_body or len(stmt.else_body.body) == 0:
  760. # No or empty else_body
  761. return True
  762. # For elif, IfStmt are stored recursively in else_body
  763. return self._is_stripped_if_stmt(stmt.else_body.body[0])
  764. def translate_module_id(self, id: str) -> str:
  765. """Return the actual, internal module id for a source text id."""
  766. if id == self.options.custom_typing_module:
  767. return "typing"
  768. return id
  769. def visit_Module(self, mod: ast3.Module) -> MypyFile:
  770. self.type_ignores = {}
  771. for ti in mod.type_ignores:
  772. parsed = parse_type_ignore_tag(ti.tag)
  773. if parsed is not None:
  774. self.type_ignores[ti.lineno] = parsed
  775. else:
  776. self.fail(message_registry.INVALID_TYPE_IGNORE, ti.lineno, -1, blocker=False)
  777. body = self.fix_function_overloads(self.translate_stmt_list(mod.body, ismodule=True))
  778. return MypyFile(body, self.imports, False, self.type_ignores)
  779. # --- stmt ---
  780. # FunctionDef(identifier name, arguments args,
  781. # stmt* body, expr* decorator_list, expr? returns, string? type_comment)
  782. # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
  783. # arg? kwarg, expr* defaults)
  784. def visit_FunctionDef(self, n: ast3.FunctionDef) -> FuncDef | Decorator:
  785. return self.do_func_def(n)
  786. # AsyncFunctionDef(identifier name, arguments args,
  787. # stmt* body, expr* decorator_list, expr? returns, string? type_comment)
  788. def visit_AsyncFunctionDef(self, n: ast3.AsyncFunctionDef) -> FuncDef | Decorator:
  789. return self.do_func_def(n, is_coroutine=True)
  790. def do_func_def(
  791. self, n: ast3.FunctionDef | ast3.AsyncFunctionDef, is_coroutine: bool = False
  792. ) -> FuncDef | Decorator:
  793. """Helper shared between visit_FunctionDef and visit_AsyncFunctionDef."""
  794. self.class_and_function_stack.append("D")
  795. no_type_check = bool(
  796. n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list)
  797. )
  798. lineno = n.lineno
  799. args = self.transform_args(n.args, lineno, no_type_check=no_type_check)
  800. if special_function_elide_names(n.name):
  801. for arg in args:
  802. arg.pos_only = True
  803. arg_kinds = [arg.kind for arg in args]
  804. arg_names = [None if arg.pos_only else arg.variable.name for arg in args]
  805. arg_types: list[Type | None] = []
  806. if no_type_check:
  807. arg_types = [None] * len(args)
  808. return_type = None
  809. elif n.type_comment is not None:
  810. try:
  811. func_type_ast = ast3_parse(n.type_comment, "<func_type>", "func_type")
  812. assert isinstance(func_type_ast, FunctionType)
  813. # for ellipsis arg
  814. if (
  815. len(func_type_ast.argtypes) == 1
  816. and isinstance(func_type_ast.argtypes[0], Constant)
  817. and func_type_ast.argtypes[0].value is Ellipsis
  818. ):
  819. if n.returns:
  820. # PEP 484 disallows both type annotations and type comments
  821. self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset)
  822. arg_types = [
  823. a.type_annotation
  824. if a.type_annotation is not None
  825. else AnyType(TypeOfAny.unannotated)
  826. for a in args
  827. ]
  828. else:
  829. # PEP 484 disallows both type annotations and type comments
  830. if n.returns or any(a.type_annotation is not None for a in args):
  831. self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset)
  832. translated_args: list[Type] = TypeConverter(
  833. self.errors, line=lineno, override_column=n.col_offset
  834. ).translate_expr_list(func_type_ast.argtypes)
  835. # Use a cast to work around `list` invariance
  836. arg_types = cast(List[Optional[Type]], translated_args)
  837. return_type = TypeConverter(self.errors, line=lineno).visit(func_type_ast.returns)
  838. # add implicit self type
  839. in_method_scope = self.class_and_function_stack[-2:] == ["C", "D"]
  840. if in_method_scope and len(arg_types) < len(args):
  841. arg_types.insert(0, AnyType(TypeOfAny.special_form))
  842. except SyntaxError:
  843. stripped_type = n.type_comment.split("#", 2)[0].strip()
  844. err_msg = message_registry.TYPE_COMMENT_SYNTAX_ERROR_VALUE.format(stripped_type)
  845. self.fail(err_msg, lineno, n.col_offset)
  846. if n.type_comment and n.type_comment[0] not in ["(", "#"]:
  847. self.note(
  848. "Suggestion: wrap argument types in parentheses", lineno, n.col_offset
  849. )
  850. arg_types = [AnyType(TypeOfAny.from_error)] * len(args)
  851. return_type = AnyType(TypeOfAny.from_error)
  852. else:
  853. arg_types = [a.type_annotation for a in args]
  854. return_type = TypeConverter(
  855. self.errors, line=n.returns.lineno if n.returns else lineno
  856. ).visit(n.returns)
  857. for arg, arg_type in zip(args, arg_types):
  858. self.set_type_optional(arg_type, arg.initializer)
  859. func_type = None
  860. if any(arg_types) or return_type:
  861. if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types):
  862. self.fail(message_registry.ELLIPSIS_WITH_OTHER_TYPEARGS, lineno, n.col_offset)
  863. elif len(arg_types) > len(arg_kinds):
  864. self.fail(
  865. message_registry.TYPE_SIGNATURE_TOO_MANY_ARGS,
  866. lineno,
  867. n.col_offset,
  868. blocker=False,
  869. )
  870. elif len(arg_types) < len(arg_kinds):
  871. self.fail(
  872. message_registry.TYPE_SIGNATURE_TOO_FEW_ARGS,
  873. lineno,
  874. n.col_offset,
  875. blocker=False,
  876. )
  877. else:
  878. func_type = CallableType(
  879. [a if a is not None else AnyType(TypeOfAny.unannotated) for a in arg_types],
  880. arg_kinds,
  881. arg_names,
  882. return_type if return_type is not None else AnyType(TypeOfAny.unannotated),
  883. _dummy_fallback,
  884. )
  885. # End position is always the same.
  886. end_line = getattr(n, "end_lineno", None)
  887. end_column = getattr(n, "end_col_offset", None)
  888. self.class_and_function_stack.pop()
  889. self.class_and_function_stack.append("F")
  890. body = self.as_required_block(n.body, can_strip=True, is_coroutine=is_coroutine)
  891. func_def = FuncDef(n.name, args, body, func_type)
  892. if isinstance(func_def.type, CallableType):
  893. # semanal.py does some in-place modifications we want to avoid
  894. func_def.unanalyzed_type = func_def.type.copy_modified()
  895. if is_coroutine:
  896. func_def.is_coroutine = True
  897. if func_type is not None:
  898. func_type.definition = func_def
  899. func_type.line = lineno
  900. if n.decorator_list:
  901. # Set deco_line to the old pre-3.8 lineno, in order to keep
  902. # existing "# type: ignore" comments working:
  903. deco_line = n.decorator_list[0].lineno
  904. var = Var(func_def.name)
  905. var.is_ready = False
  906. var.set_line(lineno)
  907. func_def.is_decorated = True
  908. func_def.deco_line = deco_line
  909. func_def.set_line(lineno, n.col_offset, end_line, end_column)
  910. deco = Decorator(func_def, self.translate_expr_list(n.decorator_list), var)
  911. first = n.decorator_list[0]
  912. deco.set_line(first.lineno, first.col_offset, end_line, end_column)
  913. retval: FuncDef | Decorator = deco
  914. else:
  915. # FuncDef overrides set_line -- can't use self.set_line
  916. func_def.set_line(lineno, n.col_offset, end_line, end_column)
  917. retval = func_def
  918. self.class_and_function_stack.pop()
  919. return retval
  920. def set_type_optional(self, type: Type | None, initializer: Expression | None) -> None:
  921. if not self.options.implicit_optional:
  922. return
  923. # Indicate that type should be wrapped in an Optional if arg is initialized to None.
  924. optional = isinstance(initializer, NameExpr) and initializer.name == "None"
  925. if isinstance(type, UnboundType):
  926. type.optional = optional
  927. def transform_args(
  928. self, args: ast3.arguments, line: int, no_type_check: bool = False
  929. ) -> list[Argument]:
  930. new_args = []
  931. names: list[ast3.arg] = []
  932. posonlyargs = getattr(args, "posonlyargs", cast(List[ast3.arg], []))
  933. args_args = posonlyargs + args.args
  934. args_defaults = args.defaults
  935. num_no_defaults = len(args_args) - len(args_defaults)
  936. # positional arguments without defaults
  937. for i, a in enumerate(args_args[:num_no_defaults]):
  938. pos_only = i < len(posonlyargs)
  939. new_args.append(self.make_argument(a, None, ARG_POS, no_type_check, pos_only))
  940. names.append(a)
  941. # positional arguments with defaults
  942. for i, (a, d) in enumerate(zip(args_args[num_no_defaults:], args_defaults)):
  943. pos_only = num_no_defaults + i < len(posonlyargs)
  944. new_args.append(self.make_argument(a, d, ARG_OPT, no_type_check, pos_only))
  945. names.append(a)
  946. # *arg
  947. if args.vararg is not None:
  948. new_args.append(self.make_argument(args.vararg, None, ARG_STAR, no_type_check))
  949. names.append(args.vararg)
  950. # keyword-only arguments with defaults
  951. for a, kd in zip(args.kwonlyargs, args.kw_defaults):
  952. new_args.append(
  953. self.make_argument(
  954. a, kd, ARG_NAMED if kd is None else ARG_NAMED_OPT, no_type_check
  955. )
  956. )
  957. names.append(a)
  958. # **kwarg
  959. if args.kwarg is not None:
  960. new_args.append(self.make_argument(args.kwarg, None, ARG_STAR2, no_type_check))
  961. names.append(args.kwarg)
  962. check_arg_names([arg.variable.name for arg in new_args], names, self.fail_arg)
  963. return new_args
  964. def make_argument(
  965. self,
  966. arg: ast3.arg,
  967. default: ast3.expr | None,
  968. kind: ArgKind,
  969. no_type_check: bool,
  970. pos_only: bool = False,
  971. ) -> Argument:
  972. if no_type_check:
  973. arg_type = None
  974. else:
  975. annotation = arg.annotation
  976. type_comment = arg.type_comment
  977. if annotation is not None and type_comment is not None:
  978. self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, arg.lineno, arg.col_offset)
  979. arg_type = None
  980. if annotation is not None:
  981. arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation)
  982. else:
  983. arg_type = self.translate_type_comment(arg, type_comment)
  984. if argument_elide_name(arg.arg):
  985. pos_only = True
  986. argument = Argument(Var(arg.arg), arg_type, self.visit(default), kind, pos_only)
  987. argument.set_line(
  988. arg.lineno,
  989. arg.col_offset,
  990. getattr(arg, "end_lineno", None),
  991. getattr(arg, "end_col_offset", None),
  992. )
  993. return argument
  994. def fail_arg(self, msg: str, arg: ast3.arg) -> None:
  995. self.fail(ErrorMessage(msg), arg.lineno, arg.col_offset)
  996. # ClassDef(identifier name,
  997. # expr* bases,
  998. # keyword* keywords,
  999. # stmt* body,
  1000. # expr* decorator_list)
  1001. def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef:
  1002. self.class_and_function_stack.append("C")
  1003. keywords = [(kw.arg, self.visit(kw.value)) for kw in n.keywords if kw.arg]
  1004. cdef = ClassDef(
  1005. n.name,
  1006. self.as_required_block(n.body),
  1007. None,
  1008. self.translate_expr_list(n.bases),
  1009. metaclass=dict(keywords).get("metaclass"),
  1010. keywords=keywords,
  1011. )
  1012. cdef.decorators = self.translate_expr_list(n.decorator_list)
  1013. # Set lines to match the old mypy 0.700 lines, in order to keep
  1014. # existing "# type: ignore" comments working:
  1015. cdef.line = n.lineno
  1016. cdef.deco_line = n.decorator_list[0].lineno if n.decorator_list else None
  1017. cdef.column = n.col_offset
  1018. cdef.end_line = getattr(n, "end_lineno", None)
  1019. cdef.end_column = getattr(n, "end_col_offset", None)
  1020. self.class_and_function_stack.pop()
  1021. return cdef
  1022. # Return(expr? value)
  1023. def visit_Return(self, n: ast3.Return) -> ReturnStmt:
  1024. node = ReturnStmt(self.visit(n.value))
  1025. return self.set_line(node, n)
  1026. # Delete(expr* targets)
  1027. def visit_Delete(self, n: ast3.Delete) -> DelStmt:
  1028. if len(n.targets) > 1:
  1029. tup = TupleExpr(self.translate_expr_list(n.targets))
  1030. tup.set_line(n.lineno)
  1031. node = DelStmt(tup)
  1032. else:
  1033. node = DelStmt(self.visit(n.targets[0]))
  1034. return self.set_line(node, n)
  1035. # Assign(expr* targets, expr? value, string? type_comment, expr? annotation)
  1036. def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt:
  1037. lvalues = self.translate_expr_list(n.targets)
  1038. rvalue = self.visit(n.value)
  1039. typ = self.translate_type_comment(n, n.type_comment)
  1040. s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False)
  1041. return self.set_line(s, n)
  1042. # AnnAssign(expr target, expr annotation, expr? value, int simple)
  1043. def visit_AnnAssign(self, n: ast3.AnnAssign) -> AssignmentStmt:
  1044. line = n.lineno
  1045. if n.value is None: # always allow 'x: int'
  1046. rvalue: Expression = TempNode(AnyType(TypeOfAny.special_form), no_rhs=True)
  1047. rvalue.line = line
  1048. rvalue.column = n.col_offset
  1049. else:
  1050. rvalue = self.visit(n.value)
  1051. typ = TypeConverter(self.errors, line=line).visit(n.annotation)
  1052. assert typ is not None
  1053. typ.column = n.annotation.col_offset
  1054. s = AssignmentStmt([self.visit(n.target)], rvalue, type=typ, new_syntax=True)
  1055. return self.set_line(s, n)
  1056. # AugAssign(expr target, operator op, expr value)
  1057. def visit_AugAssign(self, n: ast3.AugAssign) -> OperatorAssignmentStmt:
  1058. s = OperatorAssignmentStmt(
  1059. self.from_operator(n.op), self.visit(n.target), self.visit(n.value)
  1060. )
  1061. return self.set_line(s, n)
  1062. # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
  1063. def visit_For(self, n: ast3.For) -> ForStmt:
  1064. target_type = self.translate_type_comment(n, n.type_comment)
  1065. node = ForStmt(
  1066. self.visit(n.target),
  1067. self.visit(n.iter),
  1068. self.as_required_block(n.body),
  1069. self.as_block(n.orelse),
  1070. target_type,
  1071. )
  1072. return self.set_line(node, n)
  1073. # AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
  1074. def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt:
  1075. target_type = self.translate_type_comment(n, n.type_comment)
  1076. node = ForStmt(
  1077. self.visit(n.target),
  1078. self.visit(n.iter),
  1079. self.as_required_block(n.body),
  1080. self.as_block(n.orelse),
  1081. target_type,
  1082. )
  1083. node.is_async = True
  1084. return self.set_line(node, n)
  1085. # While(expr test, stmt* body, stmt* orelse)
  1086. def visit_While(self, n: ast3.While) -> WhileStmt:
  1087. node = WhileStmt(
  1088. self.visit(n.test), self.as_required_block(n.body), self.as_block(n.orelse)
  1089. )
  1090. return self.set_line(node, n)
  1091. # If(expr test, stmt* body, stmt* orelse)
  1092. def visit_If(self, n: ast3.If) -> IfStmt:
  1093. node = IfStmt(
  1094. [self.visit(n.test)], [self.as_required_block(n.body)], self.as_block(n.orelse)
  1095. )
  1096. return self.set_line(node, n)
  1097. # With(withitem* items, stmt* body, string? type_comment)
  1098. def visit_With(self, n: ast3.With) -> WithStmt:
  1099. target_type = self.translate_type_comment(n, n.type_comment)
  1100. node = WithStmt(
  1101. [self.visit(i.context_expr) for i in n.items],
  1102. [self.visit(i.optional_vars) for i in n.items],
  1103. self.as_required_block(n.body),
  1104. target_type,
  1105. )
  1106. return self.set_line(node, n)
  1107. # AsyncWith(withitem* items, stmt* body, string? type_comment)
  1108. def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt:
  1109. target_type = self.translate_type_comment(n, n.type_comment)
  1110. s = WithStmt(
  1111. [self.visit(i.context_expr) for i in n.items],
  1112. [self.visit(i.optional_vars) for i in n.items],
  1113. self.as_required_block(n.body),
  1114. target_type,
  1115. )
  1116. s.is_async = True
  1117. return self.set_line(s, n)
  1118. # Raise(expr? exc, expr? cause)
  1119. def visit_Raise(self, n: ast3.Raise) -> RaiseStmt:
  1120. node = RaiseStmt(self.visit(n.exc), self.visit(n.cause))
  1121. return self.set_line(node, n)
  1122. # Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
  1123. def visit_Try(self, n: ast3.Try) -> TryStmt:
  1124. vs = [
  1125. self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers
  1126. ]
  1127. types = [self.visit(h.type) for h in n.handlers]
  1128. handlers = [self.as_required_block(h.body) for h in n.handlers]
  1129. node = TryStmt(
  1130. self.as_required_block(n.body),
  1131. vs,
  1132. types,
  1133. handlers,
  1134. self.as_block(n.orelse),
  1135. self.as_block(n.finalbody),
  1136. )
  1137. return self.set_line(node, n)
  1138. def visit_TryStar(self, n: TryStar) -> TryStmt:
  1139. vs = [
  1140. self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers
  1141. ]
  1142. types = [self.visit(h.type) for h in n.handlers]
  1143. handlers = [self.as_required_block(h.body) for h in n.handlers]
  1144. node = TryStmt(
  1145. self.as_required_block(n.body),
  1146. vs,
  1147. types,
  1148. handlers,
  1149. self.as_block(n.orelse),
  1150. self.as_block(n.finalbody),
  1151. )
  1152. node.is_star = True
  1153. return self.set_line(node, n)
  1154. # Assert(expr test, expr? msg)
  1155. def visit_Assert(self, n: ast3.Assert) -> AssertStmt:
  1156. node = AssertStmt(self.visit(n.test), self.visit(n.msg))
  1157. return self.set_line(node, n)
  1158. # Import(alias* names)
  1159. def visit_Import(self, n: ast3.Import) -> Import:
  1160. names: list[tuple[str, str | None]] = []
  1161. for alias in n.names:
  1162. name = self.translate_module_id(alias.name)
  1163. asname = alias.asname
  1164. if asname is None and name != alias.name:
  1165. # if the module name has been translated (and it's not already
  1166. # an explicit import-as), make it an implicit import-as the
  1167. # original name
  1168. asname = alias.name
  1169. names.append((name, asname))
  1170. i = Import(names)
  1171. self.imports.append(i)
  1172. return self.set_line(i, n)
  1173. # ImportFrom(identifier? module, alias* names, int? level)
  1174. def visit_ImportFrom(self, n: ast3.ImportFrom) -> ImportBase:
  1175. assert n.level is not None
  1176. if len(n.names) == 1 and n.names[0].name == "*":
  1177. mod = n.module if n.module is not None else ""
  1178. i: ImportBase = ImportAll(mod, n.level)
  1179. else:
  1180. i = ImportFrom(
  1181. self.translate_module_id(n.module) if n.module is not None else "",
  1182. n.level,
  1183. [(a.name, a.asname) for a in n.names],
  1184. )
  1185. self.imports.append(i)
  1186. return self.set_line(i, n)
  1187. # Global(identifier* names)
  1188. def visit_Global(self, n: ast3.Global) -> GlobalDecl:
  1189. g = GlobalDecl(n.names)
  1190. return self.set_line(g, n)
  1191. # Nonlocal(identifier* names)
  1192. def visit_Nonlocal(self, n: ast3.Nonlocal) -> NonlocalDecl:
  1193. d = NonlocalDecl(n.names)
  1194. return self.set_line(d, n)
  1195. # Expr(expr value)
  1196. def visit_Expr(self, n: ast3.Expr) -> ExpressionStmt:
  1197. value = self.visit(n.value)
  1198. node = ExpressionStmt(value)
  1199. return self.set_line(node, n)
  1200. # Pass
  1201. def visit_Pass(self, n: ast3.Pass) -> PassStmt:
  1202. s = PassStmt()
  1203. return self.set_line(s, n)
  1204. # Break
  1205. def visit_Break(self, n: ast3.Break) -> BreakStmt:
  1206. s = BreakStmt()
  1207. return self.set_line(s, n)
  1208. # Continue
  1209. def visit_Continue(self, n: ast3.Continue) -> ContinueStmt:
  1210. s = ContinueStmt()
  1211. return self.set_line(s, n)
  1212. # --- expr ---
  1213. def visit_NamedExpr(self, n: NamedExpr) -> AssignmentExpr:
  1214. s = AssignmentExpr(self.visit(n.target), self.visit(n.value))
  1215. return self.set_line(s, n)
  1216. # BoolOp(boolop op, expr* values)
  1217. def visit_BoolOp(self, n: ast3.BoolOp) -> OpExpr:
  1218. # mypy translates (1 and 2 and 3) as (1 and (2 and 3))
  1219. assert len(n.values) >= 2
  1220. op_node = n.op
  1221. if isinstance(op_node, ast3.And):
  1222. op = "and"
  1223. elif isinstance(op_node, ast3.Or):
  1224. op = "or"
  1225. else:
  1226. raise RuntimeError("unknown BoolOp " + str(type(n)))
  1227. # potentially inefficient!
  1228. return self.group(op, self.translate_expr_list(n.values), n)
  1229. def group(self, op: str, vals: list[Expression], n: ast3.expr) -> OpExpr:
  1230. if len(vals) == 2:
  1231. e = OpExpr(op, vals[0], vals[1])
  1232. else:
  1233. e = OpExpr(op, vals[0], self.group(op, vals[1:], n))
  1234. return self.set_line(e, n)
  1235. # BinOp(expr left, operator op, expr right)
  1236. def visit_BinOp(self, n: ast3.BinOp) -> OpExpr:
  1237. op = self.from_operator(n.op)
  1238. if op is None:
  1239. raise RuntimeError("cannot translate BinOp " + str(type(n.op)))
  1240. e = OpExpr(op, self.visit(n.left), self.visit(n.right))
  1241. return self.set_line(e, n)
  1242. # UnaryOp(unaryop op, expr operand)
  1243. def visit_UnaryOp(self, n: ast3.UnaryOp) -> UnaryExpr:
  1244. op = None
  1245. if isinstance(n.op, ast3.Invert):
  1246. op = "~"
  1247. elif isinstance(n.op, ast3.Not):
  1248. op = "not"
  1249. elif isinstance(n.op, ast3.UAdd):
  1250. op = "+"
  1251. elif isinstance(n.op, ast3.USub):
  1252. op = "-"
  1253. if op is None:
  1254. raise RuntimeError("cannot translate UnaryOp " + str(type(n.op)))
  1255. e = UnaryExpr(op, self.visit(n.operand))
  1256. return self.set_line(e, n)
  1257. # Lambda(arguments args, expr body)
  1258. def visit_Lambda(self, n: ast3.Lambda) -> LambdaExpr:
  1259. body = ast3.Return(n.body)
  1260. body.lineno = n.body.lineno
  1261. body.col_offset = n.body.col_offset
  1262. self.class_and_function_stack.append("L")
  1263. e = LambdaExpr(self.transform_args(n.args, n.lineno), self.as_required_block([body]))
  1264. self.class_and_function_stack.pop()
  1265. e.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line
  1266. return e
  1267. # IfExp(expr test, expr body, expr orelse)
  1268. def visit_IfExp(self, n: ast3.IfExp) -> ConditionalExpr:
  1269. e = ConditionalExpr(self.visit(n.test), self.visit(n.body), self.visit(n.orelse))
  1270. return self.set_line(e, n)
  1271. # Dict(expr* keys, expr* values)
  1272. def visit_Dict(self, n: ast3.Dict) -> DictExpr:
  1273. e = DictExpr(
  1274. list(zip(self.translate_opt_expr_list(n.keys), self.translate_expr_list(n.values)))
  1275. )
  1276. return self.set_line(e, n)
  1277. # Set(expr* elts)
  1278. def visit_Set(self, n: ast3.Set) -> SetExpr:
  1279. e = SetExpr(self.translate_expr_list(n.elts))
  1280. return self.set_line(e, n)
  1281. # ListComp(expr elt, comprehension* generators)
  1282. def visit_ListComp(self, n: ast3.ListComp) -> ListComprehension:
  1283. e = ListComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n)))
  1284. return self.set_line(e, n)
  1285. # SetComp(expr elt, comprehension* generators)
  1286. def visit_SetComp(self, n: ast3.SetComp) -> SetComprehension:
  1287. e = SetComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n)))
  1288. return self.set_line(e, n)
  1289. # DictComp(expr key, expr value, comprehension* generators)
  1290. def visit_DictComp(self, n: ast3.DictComp) -> DictionaryComprehension:
  1291. targets = [self.visit(c.target) for c in n.generators]
  1292. iters = [self.visit(c.iter) for c in n.generators]
  1293. ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators]
  1294. is_async = [bool(c.is_async) for c in n.generators]
  1295. e = DictionaryComprehension(
  1296. self.visit(n.key), self.visit(n.value), targets, iters, ifs_list, is_async
  1297. )
  1298. return self.set_line(e, n)
  1299. # GeneratorExp(expr elt, comprehension* generators)
  1300. def visit_GeneratorExp(self, n: ast3.GeneratorExp) -> GeneratorExpr:
  1301. targets = [self.visit(c.target) for c in n.generators]
  1302. iters = [self.visit(c.iter) for c in n.generators]
  1303. ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators]
  1304. is_async = [bool(c.is_async) for c in n.generators]
  1305. e = GeneratorExpr(self.visit(n.elt), targets, iters, ifs_list, is_async)
  1306. return self.set_line(e, n)
  1307. # Await(expr value)
  1308. def visit_Await(self, n: ast3.Await) -> AwaitExpr:
  1309. v = self.visit(n.value)
  1310. e = AwaitExpr(v)
  1311. return self.set_line(e, n)
  1312. # Yield(expr? value)
  1313. def visit_Yield(self, n: ast3.Yield) -> YieldExpr:
  1314. e = YieldExpr(self.visit(n.value))
  1315. return self.set_line(e, n)
  1316. # YieldFrom(expr value)
  1317. def visit_YieldFrom(self, n: ast3.YieldFrom) -> YieldFromExpr:
  1318. e = YieldFromExpr(self.visit(n.value))
  1319. return self.set_line(e, n)
  1320. # Compare(expr left, cmpop* ops, expr* comparators)
  1321. def visit_Compare(self, n: ast3.Compare) -> ComparisonExpr:
  1322. operators = [self.from_comp_operator(o) for o in n.ops]
  1323. operands = self.translate_expr_list([n.left] + n.comparators)
  1324. e = ComparisonExpr(operators, operands)
  1325. return self.set_line(e, n)
  1326. # Call(expr func, expr* args, keyword* keywords)
  1327. # keyword = (identifier? arg, expr value)
  1328. def visit_Call(self, n: Call) -> CallExpr:
  1329. args = n.args
  1330. keywords = n.keywords
  1331. keyword_names = [k.arg for k in keywords]
  1332. arg_types = self.translate_expr_list(
  1333. [a.value if isinstance(a, Starred) else a for a in args] + [k.value for k in keywords]
  1334. )
  1335. arg_kinds = [ARG_STAR if type(a) is Starred else ARG_POS for a in args] + [
  1336. ARG_STAR2 if arg is None else ARG_NAMED for arg in keyword_names
  1337. ]
  1338. e = CallExpr(
  1339. self.visit(n.func),
  1340. arg_types,
  1341. arg_kinds,
  1342. cast("List[Optional[str]]", [None] * len(args)) + keyword_names,
  1343. )
  1344. return self.set_line(e, n)
  1345. # Constant(object value) -- a constant, in Python 3.8.
  1346. def visit_Constant(self, n: Constant) -> Any:
  1347. val = n.value
  1348. e: Any = None
  1349. if val is None:
  1350. e = NameExpr("None")
  1351. elif isinstance(val, str):
  1352. e = StrExpr(val)
  1353. elif isinstance(val, bytes):
  1354. e = BytesExpr(bytes_to_human_readable_repr(val))
  1355. elif isinstance(val, bool): # Must check before int!
  1356. e = NameExpr(str(val))
  1357. elif isinstance(val, int):
  1358. e = IntExpr(val)
  1359. elif isinstance(val, float):
  1360. e = FloatExpr(val)
  1361. elif isinstance(val, complex):
  1362. e = ComplexExpr(val)
  1363. elif val is Ellipsis:
  1364. e = EllipsisExpr()
  1365. else:
  1366. raise RuntimeError("Constant not implemented for " + str(type(val)))
  1367. return self.set_line(e, n)
  1368. # JoinedStr(expr* values)
  1369. def visit_JoinedStr(self, n: ast3.JoinedStr) -> Expression:
  1370. # Each of n.values is a str or FormattedValue; we just concatenate
  1371. # them all using ''.join.
  1372. empty_string = StrExpr("")
  1373. empty_string.set_line(n.lineno, n.col_offset)
  1374. strs_to_join = ListExpr(self.translate_expr_list(n.values))
  1375. strs_to_join.set_line(empty_string)
  1376. # Don't make unnecessary join call if there is only one str to join
  1377. if len(strs_to_join.items) == 1:
  1378. return self.set_line(strs_to_join.items[0], n)
  1379. elif len(strs_to_join.items) > 1:
  1380. last = strs_to_join.items[-1]
  1381. if isinstance(last, StrExpr) and last.value == "":
  1382. # 3.12 can add an empty literal at the end. Delete it for consistency
  1383. # between Python versions.
  1384. del strs_to_join.items[-1:]
  1385. join_method = MemberExpr(empty_string, "join")
  1386. join_method.set_line(empty_string)
  1387. result_expression = CallExpr(join_method, [strs_to_join], [ARG_POS], [None])
  1388. return self.set_line(result_expression, n)
  1389. # FormattedValue(expr value)
  1390. def visit_FormattedValue(self, n: ast3.FormattedValue) -> Expression:
  1391. # A FormattedValue is a component of a JoinedStr, or it can exist
  1392. # on its own. We translate them to individual '{}'.format(value)
  1393. # calls. Format specifier and conversion information is passed along
  1394. # to allow mypyc to support f-strings with format specifiers and conversions.
  1395. val_exp = self.visit(n.value)
  1396. val_exp.set_line(n.lineno, n.col_offset)
  1397. conv_str = "" if n.conversion < 0 else "!" + chr(n.conversion)
  1398. format_string = StrExpr("{" + conv_str + ":{}}")
  1399. format_spec_exp = self.visit(n.format_spec) if n.format_spec is not None else StrExpr("")
  1400. format_string.set_line(n.lineno, n.col_offset)
  1401. format_method = MemberExpr(format_string, "format")
  1402. format_method.set_line(format_string)
  1403. result_expression = CallExpr(
  1404. format_method, [val_exp, format_spec_exp], [ARG_POS, ARG_POS], [None, None]
  1405. )
  1406. return self.set_line(result_expression, n)
  1407. # Attribute(expr value, identifier attr, expr_context ctx)
  1408. def visit_Attribute(self, n: Attribute) -> MemberExpr | SuperExpr:
  1409. value = n.value
  1410. member_expr = MemberExpr(self.visit(value), n.attr)
  1411. obj = member_expr.expr
  1412. if (
  1413. isinstance(obj, CallExpr)
  1414. and isinstance(obj.callee, NameExpr)
  1415. and obj.callee.name == "super"
  1416. ):
  1417. e: MemberExpr | SuperExpr = SuperExpr(member_expr.name, obj)
  1418. else:
  1419. e = member_expr
  1420. return self.set_line(e, n)
  1421. # Subscript(expr value, slice slice, expr_context ctx)
  1422. def visit_Subscript(self, n: ast3.Subscript) -> IndexExpr:
  1423. e = IndexExpr(self.visit(n.value), self.visit(n.slice))
  1424. self.set_line(e, n)
  1425. # alias to please mypyc
  1426. is_py38_or_earlier = sys.version_info < (3, 9)
  1427. if isinstance(n.slice, ast3.Slice) or (
  1428. is_py38_or_earlier and isinstance(n.slice, ast3.ExtSlice)
  1429. ):
  1430. # Before Python 3.9, Slice has no line/column in the raw ast. To avoid incompatibility
  1431. # visit_Slice doesn't set_line, even in Python 3.9 on.
  1432. # ExtSlice also has no line/column info. In Python 3.9 on, line/column is set for
  1433. # e.index when visiting n.slice.
  1434. e.index.line = e.line
  1435. e.index.column = e.column
  1436. return e
  1437. # Starred(expr value, expr_context ctx)
  1438. def visit_Starred(self, n: Starred) -> StarExpr:
  1439. e = StarExpr(self.visit(n.value))
  1440. return self.set_line(e, n)
  1441. # Name(identifier id, expr_context ctx)
  1442. def visit_Name(self, n: Name) -> NameExpr:
  1443. e = NameExpr(n.id)
  1444. return self.set_line(e, n)
  1445. # List(expr* elts, expr_context ctx)
  1446. def visit_List(self, n: ast3.List) -> ListExpr | TupleExpr:
  1447. expr_list: list[Expression] = [self.visit(e) for e in n.elts]
  1448. if isinstance(n.ctx, ast3.Store):
  1449. # [x, y] = z and (x, y) = z means exactly the same thing
  1450. e: ListExpr | TupleExpr = TupleExpr(expr_list)
  1451. else:
  1452. e = ListExpr(expr_list)
  1453. return self.set_line(e, n)
  1454. # Tuple(expr* elts, expr_context ctx)
  1455. def visit_Tuple(self, n: ast3.Tuple) -> TupleExpr:
  1456. e = TupleExpr(self.translate_expr_list(n.elts))
  1457. return self.set_line(e, n)
  1458. # --- slice ---
  1459. # Slice(expr? lower, expr? upper, expr? step)
  1460. def visit_Slice(self, n: ast3.Slice) -> SliceExpr:
  1461. return SliceExpr(self.visit(n.lower), self.visit(n.upper), self.visit(n.step))
  1462. # ExtSlice(slice* dims)
  1463. def visit_ExtSlice(self, n: ast3.ExtSlice) -> TupleExpr:
  1464. # cast for mypyc's benefit on Python 3.9
  1465. return TupleExpr(self.translate_expr_list(cast(Any, n).dims))
  1466. # Index(expr value)
  1467. def visit_Index(self, n: Index) -> Node:
  1468. # cast for mypyc's benefit on Python 3.9
  1469. value = self.visit(cast(Any, n).value)
  1470. assert isinstance(value, Node)
  1471. return value
  1472. # Match(expr subject, match_case* cases) # python 3.10 and later
  1473. def visit_Match(self, n: Match) -> MatchStmt:
  1474. node = MatchStmt(
  1475. self.visit(n.subject),
  1476. [self.visit(c.pattern) for c in n.cases],
  1477. [self.visit(c.guard) for c in n.cases],
  1478. [self.as_required_block(c.body) for c in n.cases],
  1479. )
  1480. return self.set_line(node, n)
  1481. def visit_MatchValue(self, n: MatchValue) -> ValuePattern:
  1482. node = ValuePattern(self.visit(n.value))
  1483. return self.set_line(node, n)
  1484. def visit_MatchSingleton(self, n: MatchSingleton) -> SingletonPattern:
  1485. node = SingletonPattern(n.value)
  1486. return self.set_line(node, n)
  1487. def visit_MatchSequence(self, n: MatchSequence) -> SequencePattern:
  1488. patterns = [self.visit(p) for p in n.patterns]
  1489. stars = [p for p in patterns if isinstance(p, StarredPattern)]
  1490. assert len(stars) < 2
  1491. node = SequencePattern(patterns)
  1492. return self.set_line(node, n)
  1493. def visit_MatchStar(self, n: MatchStar) -> StarredPattern:
  1494. if n.name is None:
  1495. node = StarredPattern(None)
  1496. else:
  1497. name = self.set_line(NameExpr(n.name), n)
  1498. node = StarredPattern(name)
  1499. return self.set_line(node, n)
  1500. def visit_MatchMapping(self, n: MatchMapping) -> MappingPattern:
  1501. keys = [self.visit(k) for k in n.keys]
  1502. values = [self.visit(v) for v in n.patterns]
  1503. if n.rest is None:
  1504. rest = None
  1505. else:
  1506. rest = NameExpr(n.rest)
  1507. node = MappingPattern(keys, values, rest)
  1508. return self.set_line(node, n)
  1509. def visit_MatchClass(self, n: MatchClass) -> ClassPattern:
  1510. class_ref = self.visit(n.cls)
  1511. assert isinstance(class_ref, RefExpr)
  1512. positionals = [self.visit(p) for p in n.patterns]
  1513. keyword_keys = n.kwd_attrs
  1514. keyword_values = [self.visit(p) for p in n.kwd_patterns]
  1515. node = ClassPattern(class_ref, positionals, keyword_keys, keyword_values)
  1516. return self.set_line(node, n)
  1517. # MatchAs(expr pattern, identifier name)
  1518. def visit_MatchAs(self, n: MatchAs) -> AsPattern:
  1519. if n.name is None:
  1520. name = None
  1521. else:
  1522. name = NameExpr(n.name)
  1523. name = self.set_line(name, n)
  1524. node = AsPattern(self.visit(n.pattern), name)
  1525. return self.set_line(node, n)
  1526. # MatchOr(expr* pattern)
  1527. def visit_MatchOr(self, n: MatchOr) -> OrPattern:
  1528. node = OrPattern([self.visit(pattern) for pattern in n.patterns])
  1529. return self.set_line(node, n)
  1530. class TypeConverter:
  1531. def __init__(
  1532. self,
  1533. errors: Errors | None,
  1534. line: int = -1,
  1535. override_column: int = -1,
  1536. is_evaluated: bool = True,
  1537. ) -> None:
  1538. self.errors = errors
  1539. self.line = line
  1540. self.override_column = override_column
  1541. self.node_stack: list[AST] = []
  1542. self.is_evaluated = is_evaluated
  1543. def convert_column(self, column: int) -> int:
  1544. """Apply column override if defined; otherwise return column.
  1545. Column numbers are sometimes incorrect in the AST and the column
  1546. override can be used to work around that.
  1547. """
  1548. if self.override_column < 0:
  1549. return column
  1550. else:
  1551. return self.override_column
  1552. def invalid_type(self, node: AST, note: str | None = None) -> RawExpressionType:
  1553. """Constructs a type representing some expression that normally forms an invalid type.
  1554. For example, if we see a type hint that says "3 + 4", we would transform that
  1555. expression into a RawExpressionType.
  1556. The semantic analysis layer will report an "Invalid type" error when it
  1557. encounters this type, along with the given note if one is provided.
  1558. See RawExpressionType's docstring for more details on how it's used.
  1559. """
  1560. return RawExpressionType(
  1561. None, "typing.Any", line=self.line, column=getattr(node, "col_offset", -1), note=note
  1562. )
  1563. @overload
  1564. def visit(self, node: ast3.expr) -> ProperType:
  1565. ...
  1566. @overload
  1567. def visit(self, node: AST | None) -> ProperType | None:
  1568. ...
  1569. def visit(self, node: AST | None) -> ProperType | None:
  1570. """Modified visit -- keep track of the stack of nodes"""
  1571. if node is None:
  1572. return None
  1573. self.node_stack.append(node)
  1574. try:
  1575. method = "visit_" + node.__class__.__name__
  1576. visitor = getattr(self, method, None)
  1577. if visitor is not None:
  1578. typ = visitor(node)
  1579. assert isinstance(typ, ProperType)
  1580. return typ
  1581. else:
  1582. return self.invalid_type(node)
  1583. finally:
  1584. self.node_stack.pop()
  1585. def parent(self) -> AST | None:
  1586. """Return the AST node above the one we are processing"""
  1587. if len(self.node_stack) < 2:
  1588. return None
  1589. return self.node_stack[-2]
  1590. def fail(self, msg: ErrorMessage, line: int, column: int) -> None:
  1591. if self.errors:
  1592. self.errors.report(line, column, msg.value, blocker=True, code=msg.code)
  1593. def note(self, msg: str, line: int, column: int) -> None:
  1594. if self.errors:
  1595. self.errors.report(line, column, msg, severity="note", code=codes.SYNTAX)
  1596. def translate_expr_list(self, l: Sequence[ast3.expr]) -> list[Type]:
  1597. return [self.visit(e) for e in l]
  1598. def visit_Call(self, e: Call) -> Type:
  1599. # Parse the arg constructor
  1600. f = e.func
  1601. constructor = stringify_name(f)
  1602. if not isinstance(self.parent(), ast3.List):
  1603. note = None
  1604. if constructor:
  1605. note = "Suggestion: use {0}[...] instead of {0}(...)".format(constructor)
  1606. return self.invalid_type(e, note=note)
  1607. if not constructor:
  1608. self.fail(message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED, e.lineno, e.col_offset)
  1609. name: str | None = None
  1610. default_type = AnyType(TypeOfAny.special_form)
  1611. typ: Type = default_type
  1612. for i, arg in enumerate(e.args):
  1613. if i == 0:
  1614. converted = self.visit(arg)
  1615. assert converted is not None
  1616. typ = converted
  1617. elif i == 1:
  1618. name = self._extract_argument_name(arg)
  1619. else:
  1620. self.fail(message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS, f.lineno, f.col_offset)
  1621. for k in e.keywords:
  1622. value = k.value
  1623. if k.arg == "name":
  1624. if name is not None:
  1625. self.fail(
  1626. message_registry.MULTIPLE_VALUES_FOR_NAME_KWARG.format(constructor),
  1627. f.lineno,
  1628. f.col_offset,
  1629. )
  1630. name = self._extract_argument_name(value)
  1631. elif k.arg == "type":
  1632. if typ is not default_type:
  1633. self.fail(
  1634. message_registry.MULTIPLE_VALUES_FOR_TYPE_KWARG.format(constructor),
  1635. f.lineno,
  1636. f.col_offset,
  1637. )
  1638. converted = self.visit(value)
  1639. assert converted is not None
  1640. typ = converted
  1641. else:
  1642. self.fail(
  1643. message_registry.ARG_CONSTRUCTOR_UNEXPECTED_ARG.format(k.arg),
  1644. value.lineno,
  1645. value.col_offset,
  1646. )
  1647. return CallableArgument(typ, name, constructor, e.lineno, e.col_offset)
  1648. def translate_argument_list(self, l: Sequence[ast3.expr]) -> TypeList:
  1649. return TypeList([self.visit(e) for e in l], line=self.line)
  1650. def _extract_argument_name(self, n: ast3.expr) -> str | None:
  1651. if isinstance(n, Constant) and isinstance(n.value, str):
  1652. return n.value.strip()
  1653. elif isinstance(n, Constant) and n.value is None:
  1654. return None
  1655. self.fail(
  1656. message_registry.ARG_NAME_EXPECTED_STRING_LITERAL.format(type(n).__name__),
  1657. self.line,
  1658. 0,
  1659. )
  1660. return None
  1661. def visit_Name(self, n: Name) -> Type:
  1662. return UnboundType(n.id, line=self.line, column=self.convert_column(n.col_offset))
  1663. def visit_BinOp(self, n: ast3.BinOp) -> Type:
  1664. if not isinstance(n.op, ast3.BitOr):
  1665. return self.invalid_type(n)
  1666. left = self.visit(n.left)
  1667. right = self.visit(n.right)
  1668. return UnionType(
  1669. [left, right],
  1670. line=self.line,
  1671. column=self.convert_column(n.col_offset),
  1672. is_evaluated=self.is_evaluated,
  1673. uses_pep604_syntax=True,
  1674. )
  1675. def visit_Constant(self, n: Constant) -> Type:
  1676. val = n.value
  1677. if val is None:
  1678. # None is a type.
  1679. return UnboundType("None", line=self.line)
  1680. if isinstance(val, str):
  1681. # Parse forward reference.
  1682. return parse_type_string(val, "builtins.str", self.line, n.col_offset)
  1683. if val is Ellipsis:
  1684. # '...' is valid in some types.
  1685. return EllipsisType(line=self.line)
  1686. if isinstance(val, bool):
  1687. # Special case for True/False.
  1688. return RawExpressionType(val, "builtins.bool", line=self.line)
  1689. if isinstance(val, (int, float, complex)):
  1690. return self.numeric_type(val, n)
  1691. if isinstance(val, bytes):
  1692. contents = bytes_to_human_readable_repr(val)
  1693. return RawExpressionType(contents, "builtins.bytes", self.line, column=n.col_offset)
  1694. # Everything else is invalid.
  1695. return self.invalid_type(n)
  1696. # UnaryOp(op, operand)
  1697. def visit_UnaryOp(self, n: UnaryOp) -> Type:
  1698. # We support specifically Literal[-4] and nothing else.
  1699. # For example, Literal[+4] or Literal[~6] is not supported.
  1700. typ = self.visit(n.operand)
  1701. if isinstance(typ, RawExpressionType) and isinstance(n.op, USub):
  1702. if isinstance(typ.literal_value, int):
  1703. typ.literal_value *= -1
  1704. return typ
  1705. return self.invalid_type(n)
  1706. def numeric_type(self, value: object, n: AST) -> Type:
  1707. # The node's field has the type complex, but complex isn't *really*
  1708. # a parent of int and float, and this causes isinstance below
  1709. # to think that the complex branch is always picked. Avoid
  1710. # this by throwing away the type.
  1711. if isinstance(value, int):
  1712. numeric_value: int | None = value
  1713. type_name = "builtins.int"
  1714. else:
  1715. # Other kinds of numbers (floats, complex) are not valid parameters for
  1716. # RawExpressionType so we just pass in 'None' for now. We'll report the
  1717. # appropriate error at a later stage.
  1718. numeric_value = None
  1719. type_name = f"builtins.{type(value).__name__}"
  1720. return RawExpressionType(
  1721. numeric_value, type_name, line=self.line, column=getattr(n, "col_offset", -1)
  1722. )
  1723. def visit_Index(self, n: ast3.Index) -> Type:
  1724. # cast for mypyc's benefit on Python 3.9
  1725. value = self.visit(cast(Any, n).value)
  1726. assert isinstance(value, Type)
  1727. return value
  1728. def visit_Slice(self, n: ast3.Slice) -> Type:
  1729. return self.invalid_type(n, note="did you mean to use ',' instead of ':' ?")
  1730. # Subscript(expr value, slice slice, expr_context ctx) # Python 3.8 and before
  1731. # Subscript(expr value, expr slice, expr_context ctx) # Python 3.9 and later
  1732. def visit_Subscript(self, n: ast3.Subscript) -> Type:
  1733. if sys.version_info >= (3, 9): # Really 3.9a5 or later
  1734. sliceval: Any = n.slice
  1735. # Python 3.8 or earlier use a different AST structure for subscripts
  1736. elif isinstance(n.slice, ast3.Index):
  1737. sliceval: Any = n.slice.value
  1738. elif isinstance(n.slice, ast3.Slice):
  1739. sliceval = copy.deepcopy(n.slice) # so we don't mutate passed AST
  1740. if getattr(sliceval, "col_offset", None) is None:
  1741. # Fix column information so that we get Python 3.9+ message order
  1742. sliceval.col_offset = sliceval.lower.col_offset
  1743. else:
  1744. assert isinstance(n.slice, ast3.ExtSlice)
  1745. dims = copy.deepcopy(n.slice.dims)
  1746. for s in dims:
  1747. if getattr(s, "col_offset", None) is None:
  1748. if isinstance(s, ast3.Index):
  1749. s.col_offset = s.value.col_offset
  1750. elif isinstance(s, ast3.Slice):
  1751. assert s.lower is not None
  1752. s.col_offset = s.lower.col_offset
  1753. sliceval = ast3.Tuple(dims, n.ctx)
  1754. empty_tuple_index = False
  1755. if isinstance(sliceval, ast3.Tuple):
  1756. params = self.translate_expr_list(sliceval.elts)
  1757. if len(sliceval.elts) == 0:
  1758. empty_tuple_index = True
  1759. else:
  1760. params = [self.visit(sliceval)]
  1761. value = self.visit(n.value)
  1762. if isinstance(value, UnboundType) and not value.args:
  1763. return UnboundType(
  1764. value.name,
  1765. params,
  1766. line=self.line,
  1767. column=value.column,
  1768. empty_tuple_index=empty_tuple_index,
  1769. )
  1770. else:
  1771. return self.invalid_type(n)
  1772. def visit_Tuple(self, n: ast3.Tuple) -> Type:
  1773. return TupleType(
  1774. self.translate_expr_list(n.elts),
  1775. _dummy_fallback,
  1776. implicit=True,
  1777. line=self.line,
  1778. column=self.convert_column(n.col_offset),
  1779. )
  1780. # Attribute(expr value, identifier attr, expr_context ctx)
  1781. def visit_Attribute(self, n: Attribute) -> Type:
  1782. before_dot = self.visit(n.value)
  1783. if isinstance(before_dot, UnboundType) and not before_dot.args:
  1784. return UnboundType(f"{before_dot.name}.{n.attr}", line=self.line)
  1785. else:
  1786. return self.invalid_type(n)
  1787. # List(expr* elts, expr_context ctx)
  1788. def visit_List(self, n: ast3.List) -> Type:
  1789. assert isinstance(n.ctx, ast3.Load)
  1790. return self.translate_argument_list(n.elts)
  1791. def stringify_name(n: AST) -> str | None:
  1792. if isinstance(n, Name):
  1793. return n.id
  1794. elif isinstance(n, Attribute):
  1795. sv = stringify_name(n.value)
  1796. if sv is not None:
  1797. return f"{sv}.{n.attr}"
  1798. return None # Can't do it.
  1799. class FindAttributeAssign(TraverserVisitor):
  1800. """Check if an AST contains attribute assignments (e.g. self.x = 0)."""
  1801. def __init__(self) -> None:
  1802. self.lvalue = False
  1803. self.found = False
  1804. def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
  1805. self.lvalue = True
  1806. for lv in s.lvalues:
  1807. lv.accept(self)
  1808. self.lvalue = False
  1809. def visit_with_stmt(self, s: WithStmt) -> None:
  1810. self.lvalue = True
  1811. for lv in s.target:
  1812. if lv is not None:
  1813. lv.accept(self)
  1814. self.lvalue = False
  1815. s.body.accept(self)
  1816. def visit_for_stmt(self, s: ForStmt) -> None:
  1817. self.lvalue = True
  1818. s.index.accept(self)
  1819. self.lvalue = False
  1820. s.body.accept(self)
  1821. if s.else_body:
  1822. s.else_body.accept(self)
  1823. def visit_expression_stmt(self, s: ExpressionStmt) -> None:
  1824. # No need to look inside these
  1825. pass
  1826. def visit_call_expr(self, e: CallExpr) -> None:
  1827. # No need to look inside these
  1828. pass
  1829. def visit_index_expr(self, e: IndexExpr) -> None:
  1830. # No need to look inside these
  1831. pass
  1832. def visit_member_expr(self, e: MemberExpr) -> None:
  1833. if self.lvalue:
  1834. self.found = True
  1835. class FindYield(TraverserVisitor):
  1836. """Check if an AST contains yields or yield froms."""
  1837. def __init__(self) -> None:
  1838. self.found = False
  1839. def visit_yield_expr(self, e: YieldExpr) -> None:
  1840. self.found = True
  1841. def visit_yield_from_expr(self, e: YieldFromExpr) -> None:
  1842. self.found = True
  1843. def is_possible_trivial_body(s: list[Statement]) -> bool:
  1844. """Could the statements form a "trivial" function body, such as 'pass'?
  1845. This mimics mypy.semanal.is_trivial_body, but this runs before
  1846. semantic analysis so some checks must be conservative.
  1847. """
  1848. l = len(s)
  1849. if l == 0:
  1850. return False
  1851. i = 0
  1852. if isinstance(s[0], ExpressionStmt) and isinstance(s[0].expr, StrExpr):
  1853. # Skip docstring
  1854. i += 1
  1855. if i == l:
  1856. return True
  1857. if l > i + 1:
  1858. return False
  1859. stmt = s[i]
  1860. return isinstance(stmt, (PassStmt, RaiseStmt)) or (
  1861. isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr)
  1862. )