fastparse.py 84 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232
  1. from __future__ import annotations
  2. import copy
  3. import re
  4. import sys
  5. import warnings
  6. from typing import Any, Callable, List, Optional, Sequence, TypeVar, Union, cast
  7. from typing_extensions import Final, Literal, overload
  8. from mypy import defaults, errorcodes as codes, message_registry
  9. from mypy.errors import Errors
  10. from mypy.message_registry import ErrorMessage
  11. from mypy.nodes import (
  12. ARG_NAMED,
  13. ARG_NAMED_OPT,
  14. ARG_OPT,
  15. ARG_POS,
  16. ARG_STAR,
  17. ARG_STAR2,
  18. ArgKind,
  19. Argument,
  20. AssertStmt,
  21. AssignmentExpr,
  22. AssignmentStmt,
  23. AwaitExpr,
  24. Block,
  25. BreakStmt,
  26. BytesExpr,
  27. CallExpr,
  28. ClassDef,
  29. ComparisonExpr,
  30. ComplexExpr,
  31. ConditionalExpr,
  32. ContinueStmt,
  33. Decorator,
  34. DelStmt,
  35. DictExpr,
  36. DictionaryComprehension,
  37. EllipsisExpr,
  38. Expression,
  39. ExpressionStmt,
  40. FakeInfo,
  41. FloatExpr,
  42. ForStmt,
  43. FuncDef,
  44. GeneratorExpr,
  45. GlobalDecl,
  46. IfStmt,
  47. Import,
  48. ImportAll,
  49. ImportBase,
  50. ImportFrom,
  51. IndexExpr,
  52. IntExpr,
  53. LambdaExpr,
  54. ListComprehension,
  55. ListExpr,
  56. MatchStmt,
  57. MemberExpr,
  58. MypyFile,
  59. NameExpr,
  60. Node,
  61. NonlocalDecl,
  62. OperatorAssignmentStmt,
  63. OpExpr,
  64. OverloadedFuncDef,
  65. OverloadPart,
  66. PassStmt,
  67. RaiseStmt,
  68. RefExpr,
  69. ReturnStmt,
  70. SetComprehension,
  71. SetExpr,
  72. SliceExpr,
  73. StarExpr,
  74. Statement,
  75. StrExpr,
  76. SuperExpr,
  77. TempNode,
  78. TryStmt,
  79. TupleExpr,
  80. UnaryExpr,
  81. Var,
  82. WhileStmt,
  83. WithStmt,
  84. YieldExpr,
  85. YieldFromExpr,
  86. check_arg_names,
  87. )
  88. from mypy.options import Options
  89. from mypy.patterns import (
  90. AsPattern,
  91. ClassPattern,
  92. MappingPattern,
  93. OrPattern,
  94. SequencePattern,
  95. SingletonPattern,
  96. StarredPattern,
  97. ValuePattern,
  98. )
  99. from mypy.reachability import infer_reachability_of_if_statement, mark_block_unreachable
  100. from mypy.sharedparse import argument_elide_name, special_function_elide_names
  101. from mypy.traverser import TraverserVisitor
  102. from mypy.types import (
  103. AnyType,
  104. CallableArgument,
  105. CallableType,
  106. EllipsisType,
  107. Instance,
  108. ProperType,
  109. RawExpressionType,
  110. TupleType,
  111. Type,
  112. TypeList,
  113. TypeOfAny,
  114. UnboundType,
  115. UnionType,
  116. )
  117. from mypy.util import bytes_to_human_readable_repr, unnamed_function
  118. try:
  119. # pull this into a final variable to make mypyc be quiet about the
  120. # the default argument warning
  121. PY_MINOR_VERSION: Final = sys.version_info[1]
  122. # Check if we can use the stdlib ast module instead of typed_ast.
  123. if sys.version_info >= (3, 8):
  124. import ast as ast3
  125. assert (
  126. "kind" in ast3.Constant._fields
  127. ), f"This 3.8.0 alpha ({sys.version.split()[0]}) is too old; 3.8.0a3 required"
  128. # TODO: Num, Str, Bytes, NameConstant, Ellipsis are deprecated in 3.8.
  129. # TODO: Index, ExtSlice are deprecated in 3.9.
  130. from ast import (
  131. AST,
  132. Attribute,
  133. Bytes,
  134. Call,
  135. Ellipsis as ast3_Ellipsis,
  136. Expression as ast3_Expression,
  137. FunctionType,
  138. Index,
  139. Name,
  140. NameConstant,
  141. Num,
  142. Starred,
  143. Str,
  144. UnaryOp,
  145. USub,
  146. )
  147. def ast3_parse(
  148. source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION
  149. ) -> AST:
  150. return ast3.parse(
  151. source,
  152. filename,
  153. mode,
  154. type_comments=True, # This works the magic
  155. feature_version=feature_version,
  156. )
  157. NamedExpr = ast3.NamedExpr
  158. Constant = ast3.Constant
  159. else:
  160. from typed_ast import ast3
  161. from typed_ast.ast3 import (
  162. AST,
  163. Attribute,
  164. Bytes,
  165. Call,
  166. Ellipsis as ast3_Ellipsis,
  167. Expression as ast3_Expression,
  168. FunctionType,
  169. Index,
  170. Name,
  171. NameConstant,
  172. Num,
  173. Starred,
  174. Str,
  175. UnaryOp,
  176. USub,
  177. )
  178. def ast3_parse(
  179. source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION
  180. ) -> AST:
  181. return ast3.parse(source, filename, mode, feature_version=feature_version)
  182. # These don't exist before 3.8
  183. NamedExpr = Any
  184. Constant = Any
  185. if sys.version_info >= (3, 10):
  186. Match = ast3.Match
  187. MatchValue = ast3.MatchValue
  188. MatchSingleton = ast3.MatchSingleton
  189. MatchSequence = ast3.MatchSequence
  190. MatchStar = ast3.MatchStar
  191. MatchMapping = ast3.MatchMapping
  192. MatchClass = ast3.MatchClass
  193. MatchAs = ast3.MatchAs
  194. MatchOr = ast3.MatchOr
  195. AstNode = Union[ast3.expr, ast3.stmt, ast3.pattern, ast3.ExceptHandler]
  196. else:
  197. Match = Any
  198. MatchValue = Any
  199. MatchSingleton = Any
  200. MatchSequence = Any
  201. MatchStar = Any
  202. MatchMapping = Any
  203. MatchClass = Any
  204. MatchAs = Any
  205. MatchOr = Any
  206. AstNode = Union[ast3.expr, ast3.stmt, ast3.ExceptHandler]
  207. if sys.version_info >= (3, 11):
  208. TryStar = ast3.TryStar
  209. else:
  210. TryStar = Any
  211. except ImportError:
  212. try:
  213. from typed_ast import ast35 # type: ignore[attr-defined] # noqa: F401
  214. except ImportError:
  215. print(
  216. "The typed_ast package is not installed.\n"
  217. "You can install it with `python3 -m pip install typed-ast`.",
  218. file=sys.stderr,
  219. )
  220. else:
  221. print(
  222. "You need a more recent version of the typed_ast package.\n"
  223. "You can update to the latest version with "
  224. "`python3 -m pip install -U typed-ast`.",
  225. file=sys.stderr,
  226. )
  227. sys.exit(1)
  228. N = TypeVar("N", bound=Node)
  229. # There is no way to create reasonable fallbacks at this stage,
  230. # they must be patched later.
  231. MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal")
  232. _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1)
  233. TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)")
  234. def parse(
  235. source: str | bytes,
  236. fnam: str,
  237. module: str | None,
  238. errors: Errors | None = None,
  239. options: Options | None = None,
  240. ) -> MypyFile:
  241. """Parse a source file, without doing any semantic analysis.
  242. Return the parse tree. If errors is not provided, raise ParseError
  243. on failure. Otherwise, use the errors object to report parse errors.
  244. """
  245. ignore_errors = (options is not None and options.ignore_errors) or (
  246. errors is not None and fnam in errors.ignored_files
  247. )
  248. # If errors are ignored, we can drop many function bodies to speed up type checking.
  249. strip_function_bodies = ignore_errors and (options is None or not options.preserve_asts)
  250. raise_on_error = False
  251. if options is None:
  252. options = Options()
  253. if errors is None:
  254. errors = Errors(options)
  255. raise_on_error = True
  256. errors.set_file(fnam, module, options=options)
  257. is_stub_file = fnam.endswith(".pyi")
  258. if is_stub_file:
  259. feature_version = defaults.PYTHON3_VERSION[1]
  260. if options.python_version[0] == 3 and options.python_version[1] > feature_version:
  261. feature_version = options.python_version[1]
  262. else:
  263. assert options.python_version[0] >= 3
  264. feature_version = options.python_version[1]
  265. try:
  266. # Disable deprecation warnings about \u
  267. with warnings.catch_warnings():
  268. warnings.filterwarnings("ignore", category=DeprecationWarning)
  269. ast = ast3_parse(source, fnam, "exec", feature_version=feature_version)
  270. tree = ASTConverter(
  271. options=options,
  272. is_stub=is_stub_file,
  273. errors=errors,
  274. ignore_errors=ignore_errors,
  275. strip_function_bodies=strip_function_bodies,
  276. ).visit(ast)
  277. tree.path = fnam
  278. tree.is_stub = is_stub_file
  279. except SyntaxError as e:
  280. # alias to please mypyc
  281. is_py38_or_earlier = sys.version_info < (3, 9)
  282. if is_py38_or_earlier and e.filename == "<fstring>":
  283. # In Python 3.8 and earlier, syntax errors in f-strings have lineno relative to the
  284. # start of the f-string. This would be misleading, as mypy will report the error as the
  285. # lineno within the file.
  286. e.lineno = None
  287. message = e.msg
  288. if feature_version > sys.version_info.minor and message.startswith("invalid syntax"):
  289. python_version_str = f"{options.python_version[0]}.{options.python_version[1]}"
  290. message += f"; you likely need to run mypy using Python {python_version_str} or newer"
  291. errors.report(
  292. e.lineno if e.lineno is not None else -1,
  293. e.offset,
  294. message,
  295. blocker=True,
  296. code=codes.SYNTAX,
  297. )
  298. tree = MypyFile([], [], False, {})
  299. if raise_on_error and errors.is_errors():
  300. errors.raise_error()
  301. assert isinstance(tree, MypyFile)
  302. return tree
  303. def parse_type_ignore_tag(tag: str | None) -> list[str] | None:
  304. """Parse optional "[code, ...]" tag after "# type: ignore".
  305. Return:
  306. * [] if no tag was found (ignore all errors)
  307. * list of ignored error codes if a tag was found
  308. * None if the tag was invalid.
  309. """
  310. if not tag or tag.strip() == "" or tag.strip().startswith("#"):
  311. # No tag -- ignore all errors.
  312. return []
  313. m = re.match(r"\s*\[([^]#]*)\]\s*(#.*)?$", tag)
  314. if m is None:
  315. # Invalid "# type: ignore" comment.
  316. return None
  317. return [code.strip() for code in m.group(1).split(",")]
  318. def parse_type_comment(
  319. type_comment: str, line: int, column: int, errors: Errors | None
  320. ) -> tuple[list[str] | None, ProperType | None]:
  321. """Parse type portion of a type comment (+ optional type ignore).
  322. Return (ignore info, parsed type).
  323. """
  324. try:
  325. typ = ast3_parse(type_comment, "<type_comment>", "eval")
  326. except SyntaxError:
  327. if errors is not None:
  328. stripped_type = type_comment.split("#", 2)[0].strip()
  329. err_msg = message_registry.TYPE_COMMENT_SYNTAX_ERROR_VALUE.format(stripped_type)
  330. errors.report(line, column, err_msg.value, blocker=True, code=err_msg.code)
  331. return None, None
  332. else:
  333. raise
  334. else:
  335. extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment)
  336. if extra_ignore:
  337. tag: str | None = extra_ignore.group(1)
  338. ignored: list[str] | None = parse_type_ignore_tag(tag)
  339. if ignored is None:
  340. if errors is not None:
  341. errors.report(
  342. line, column, message_registry.INVALID_TYPE_IGNORE.value, code=codes.SYNTAX
  343. )
  344. else:
  345. raise SyntaxError
  346. else:
  347. ignored = None
  348. assert isinstance(typ, ast3_Expression)
  349. converted = TypeConverter(
  350. errors, line=line, override_column=column, is_evaluated=False
  351. ).visit(typ.body)
  352. return ignored, converted
  353. def parse_type_string(
  354. expr_string: str, expr_fallback_name: str, line: int, column: int
  355. ) -> ProperType:
  356. """Parses a type that was originally present inside of an explicit string.
  357. For example, suppose we have the type `Foo["blah"]`. We should parse the
  358. string expression "blah" using this function.
  359. """
  360. try:
  361. _, node = parse_type_comment(expr_string.strip(), line=line, column=column, errors=None)
  362. if isinstance(node, UnboundType) and node.original_str_expr is None:
  363. node.original_str_expr = expr_string
  364. node.original_str_fallback = expr_fallback_name
  365. return node
  366. elif isinstance(node, UnionType):
  367. return node
  368. else:
  369. return RawExpressionType(expr_string, expr_fallback_name, line, column)
  370. except (SyntaxError, ValueError):
  371. # Note: the parser will raise a `ValueError` instead of a SyntaxError if
  372. # the string happens to contain things like \x00.
  373. return RawExpressionType(expr_string, expr_fallback_name, line, column)
  374. def is_no_type_check_decorator(expr: ast3.expr) -> bool:
  375. if isinstance(expr, Name):
  376. return expr.id == "no_type_check"
  377. elif isinstance(expr, Attribute):
  378. if isinstance(expr.value, Name):
  379. return expr.value.id == "typing" and expr.attr == "no_type_check"
  380. return False
  381. class ASTConverter:
  382. def __init__(
  383. self,
  384. options: Options,
  385. is_stub: bool,
  386. errors: Errors,
  387. *,
  388. ignore_errors: bool,
  389. strip_function_bodies: bool,
  390. ) -> None:
  391. # 'C' for class, 'D' for function signature, 'F' for function, 'L' for lambda
  392. self.class_and_function_stack: list[Literal["C", "D", "F", "L"]] = []
  393. self.imports: list[ImportBase] = []
  394. self.options = options
  395. self.is_stub = is_stub
  396. self.errors = errors
  397. self.ignore_errors = ignore_errors
  398. self.strip_function_bodies = strip_function_bodies
  399. self.type_ignores: dict[int, list[str]] = {}
  400. # Cache of visit_X methods keyed by type of visited object
  401. self.visitor_cache: dict[type, Callable[[AST | None], Any]] = {}
  402. def note(self, msg: str, line: int, column: int) -> None:
  403. self.errors.report(line, column, msg, severity="note", code=codes.SYNTAX)
  404. def fail(self, msg: ErrorMessage, line: int, column: int, blocker: bool = True) -> None:
  405. if blocker or not self.options.ignore_errors:
  406. self.errors.report(line, column, msg.value, blocker=blocker, code=msg.code)
  407. def fail_merge_overload(self, node: IfStmt) -> None:
  408. self.fail(
  409. message_registry.FAILED_TO_MERGE_OVERLOADS,
  410. line=node.line,
  411. column=node.column,
  412. blocker=False,
  413. )
  414. def visit(self, node: AST | None) -> Any:
  415. if node is None:
  416. return None
  417. typeobj = type(node)
  418. visitor = self.visitor_cache.get(typeobj)
  419. if visitor is None:
  420. method = "visit_" + node.__class__.__name__
  421. visitor = getattr(self, method)
  422. self.visitor_cache[typeobj] = visitor
  423. return visitor(node)
  424. def set_line(self, node: N, n: AstNode) -> N:
  425. node.line = n.lineno
  426. node.column = n.col_offset
  427. node.end_line = getattr(n, "end_lineno", None)
  428. node.end_column = getattr(n, "end_col_offset", None)
  429. return node
  430. def translate_opt_expr_list(self, l: Sequence[AST | None]) -> list[Expression | None]:
  431. res: list[Expression | None] = []
  432. for e in l:
  433. exp = self.visit(e)
  434. res.append(exp)
  435. return res
  436. def translate_expr_list(self, l: Sequence[AST]) -> list[Expression]:
  437. return cast(List[Expression], self.translate_opt_expr_list(l))
  438. def get_lineno(self, node: ast3.expr | ast3.stmt) -> int:
  439. if (
  440. isinstance(node, (ast3.AsyncFunctionDef, ast3.ClassDef, ast3.FunctionDef))
  441. and node.decorator_list
  442. ):
  443. return node.decorator_list[0].lineno
  444. return node.lineno
  445. def translate_stmt_list(
  446. self,
  447. stmts: Sequence[ast3.stmt],
  448. *,
  449. ismodule: bool = False,
  450. can_strip: bool = False,
  451. is_coroutine: bool = False,
  452. ) -> list[Statement]:
  453. # A "# type: ignore" comment before the first statement of a module
  454. # ignores the whole module:
  455. if (
  456. ismodule
  457. and stmts
  458. and self.type_ignores
  459. and min(self.type_ignores) < self.get_lineno(stmts[0])
  460. ):
  461. ignores = self.type_ignores[min(self.type_ignores)]
  462. if ignores:
  463. joined_ignores = ", ".join(ignores)
  464. self.fail(
  465. message_registry.TYPE_IGNORE_WITH_ERRCODE_ON_MODULE.format(joined_ignores),
  466. line=min(self.type_ignores),
  467. column=0,
  468. blocker=False,
  469. )
  470. self.errors.used_ignored_lines[self.errors.file][min(self.type_ignores)].append(
  471. codes.FILE.code
  472. )
  473. block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts)))
  474. self.set_block_lines(block, stmts)
  475. mark_block_unreachable(block)
  476. return [block]
  477. stack = self.class_and_function_stack
  478. # Fast case for stripping function bodies
  479. if (
  480. can_strip
  481. and self.strip_function_bodies
  482. and len(stack) == 1
  483. and stack[0] == "F"
  484. and not is_coroutine
  485. ):
  486. return []
  487. res: list[Statement] = []
  488. for stmt in stmts:
  489. node = self.visit(stmt)
  490. res.append(node)
  491. # Slow case for stripping function bodies
  492. if can_strip and self.strip_function_bodies:
  493. if stack[-2:] == ["C", "F"]:
  494. if is_possible_trivial_body(res):
  495. can_strip = False
  496. else:
  497. # We only strip method bodies if they don't assign to an attribute, as
  498. # this may define an attribute which has an externally visible effect.
  499. visitor = FindAttributeAssign()
  500. for s in res:
  501. s.accept(visitor)
  502. if visitor.found:
  503. can_strip = False
  504. break
  505. if can_strip and stack[-1] == "F" and is_coroutine:
  506. # Yields inside an async function affect the return type and should not
  507. # be stripped.
  508. yield_visitor = FindYield()
  509. for s in res:
  510. s.accept(yield_visitor)
  511. if yield_visitor.found:
  512. can_strip = False
  513. break
  514. if can_strip:
  515. return []
  516. return res
  517. def translate_type_comment(
  518. self, n: ast3.stmt | ast3.arg, type_comment: str | None
  519. ) -> ProperType | None:
  520. if type_comment is None:
  521. return None
  522. else:
  523. lineno = n.lineno
  524. extra_ignore, typ = parse_type_comment(type_comment, lineno, n.col_offset, self.errors)
  525. if extra_ignore is not None:
  526. self.type_ignores[lineno] = extra_ignore
  527. return typ
  528. op_map: Final[dict[type[AST], str]] = {
  529. ast3.Add: "+",
  530. ast3.Sub: "-",
  531. ast3.Mult: "*",
  532. ast3.MatMult: "@",
  533. ast3.Div: "/",
  534. ast3.Mod: "%",
  535. ast3.Pow: "**",
  536. ast3.LShift: "<<",
  537. ast3.RShift: ">>",
  538. ast3.BitOr: "|",
  539. ast3.BitXor: "^",
  540. ast3.BitAnd: "&",
  541. ast3.FloorDiv: "//",
  542. }
  543. def from_operator(self, op: ast3.operator) -> str:
  544. op_name = ASTConverter.op_map.get(type(op))
  545. if op_name is None:
  546. raise RuntimeError("Unknown operator " + str(type(op)))
  547. else:
  548. return op_name
  549. comp_op_map: Final[dict[type[AST], str]] = {
  550. ast3.Gt: ">",
  551. ast3.Lt: "<",
  552. ast3.Eq: "==",
  553. ast3.GtE: ">=",
  554. ast3.LtE: "<=",
  555. ast3.NotEq: "!=",
  556. ast3.Is: "is",
  557. ast3.IsNot: "is not",
  558. ast3.In: "in",
  559. ast3.NotIn: "not in",
  560. }
  561. def from_comp_operator(self, op: ast3.cmpop) -> str:
  562. op_name = ASTConverter.comp_op_map.get(type(op))
  563. if op_name is None:
  564. raise RuntimeError("Unknown comparison operator " + str(type(op)))
  565. else:
  566. return op_name
  567. def set_block_lines(self, b: Block, stmts: Sequence[ast3.stmt]) -> None:
  568. first, last = stmts[0], stmts[-1]
  569. b.line = first.lineno
  570. b.column = first.col_offset
  571. b.end_line = getattr(last, "end_lineno", None)
  572. b.end_column = getattr(last, "end_col_offset", None)
  573. if not b.body:
  574. return
  575. new_first = b.body[0]
  576. if isinstance(new_first, (Decorator, OverloadedFuncDef)):
  577. # Decorated function lines are different between Python versions.
  578. # copy the normalization we do for them to block first lines.
  579. b.line = new_first.line
  580. b.column = new_first.column
  581. def as_block(self, stmts: list[ast3.stmt]) -> Block | None:
  582. b = None
  583. if stmts:
  584. b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts)))
  585. self.set_block_lines(b, stmts)
  586. return b
  587. def as_required_block(
  588. self, stmts: list[ast3.stmt], *, can_strip: bool = False, is_coroutine: bool = False
  589. ) -> Block:
  590. assert stmts # must be non-empty
  591. b = Block(
  592. self.fix_function_overloads(
  593. self.translate_stmt_list(stmts, can_strip=can_strip, is_coroutine=is_coroutine)
  594. )
  595. )
  596. self.set_block_lines(b, stmts)
  597. return b
  598. def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]:
  599. ret: list[Statement] = []
  600. current_overload: list[OverloadPart] = []
  601. current_overload_name: str | None = None
  602. seen_unconditional_func_def = False
  603. last_if_stmt: IfStmt | None = None
  604. last_if_overload: Decorator | FuncDef | OverloadedFuncDef | None = None
  605. last_if_stmt_overload_name: str | None = None
  606. last_if_unknown_truth_value: IfStmt | None = None
  607. skipped_if_stmts: list[IfStmt] = []
  608. for stmt in stmts:
  609. if_overload_name: str | None = None
  610. if_block_with_overload: Block | None = None
  611. if_unknown_truth_value: IfStmt | None = None
  612. if isinstance(stmt, IfStmt) and seen_unconditional_func_def is False:
  613. # Check IfStmt block to determine if function overloads can be merged
  614. if_overload_name = self._check_ifstmt_for_overloads(stmt, current_overload_name)
  615. if if_overload_name is not None:
  616. (
  617. if_block_with_overload,
  618. if_unknown_truth_value,
  619. ) = self._get_executable_if_block_with_overloads(stmt)
  620. if (
  621. current_overload_name is not None
  622. and isinstance(stmt, (Decorator, FuncDef))
  623. and stmt.name == current_overload_name
  624. ):
  625. if last_if_stmt is not None:
  626. skipped_if_stmts.append(last_if_stmt)
  627. if last_if_overload is not None:
  628. # Last stmt was an IfStmt with same overload name
  629. # Add overloads to current_overload
  630. if isinstance(last_if_overload, OverloadedFuncDef):
  631. current_overload.extend(last_if_overload.items)
  632. else:
  633. current_overload.append(last_if_overload)
  634. last_if_stmt, last_if_overload = None, None
  635. if last_if_unknown_truth_value:
  636. self.fail_merge_overload(last_if_unknown_truth_value)
  637. last_if_unknown_truth_value = None
  638. current_overload.append(stmt)
  639. if isinstance(stmt, FuncDef):
  640. seen_unconditional_func_def = True
  641. elif (
  642. current_overload_name is not None
  643. and isinstance(stmt, IfStmt)
  644. and if_overload_name == current_overload_name
  645. ):
  646. # IfStmt only contains stmts relevant to current_overload.
  647. # Check if stmts are reachable and add them to current_overload,
  648. # otherwise skip IfStmt to allow subsequent overload
  649. # or function definitions.
  650. skipped_if_stmts.append(stmt)
  651. if if_block_with_overload is None:
  652. if if_unknown_truth_value is not None:
  653. self.fail_merge_overload(if_unknown_truth_value)
  654. continue
  655. if last_if_overload is not None:
  656. # Last stmt was an IfStmt with same overload name
  657. # Add overloads to current_overload
  658. if isinstance(last_if_overload, OverloadedFuncDef):
  659. current_overload.extend(last_if_overload.items)
  660. else:
  661. current_overload.append(last_if_overload)
  662. last_if_stmt, last_if_overload = None, None
  663. if isinstance(if_block_with_overload.body[-1], OverloadedFuncDef):
  664. skipped_if_stmts.extend(cast(List[IfStmt], if_block_with_overload.body[:-1]))
  665. current_overload.extend(if_block_with_overload.body[-1].items)
  666. else:
  667. current_overload.append(
  668. cast(Union[Decorator, FuncDef], if_block_with_overload.body[0])
  669. )
  670. else:
  671. if last_if_stmt is not None:
  672. ret.append(last_if_stmt)
  673. last_if_stmt_overload_name = current_overload_name
  674. last_if_stmt, last_if_overload = None, None
  675. last_if_unknown_truth_value = None
  676. if current_overload and current_overload_name == last_if_stmt_overload_name:
  677. # Remove last stmt (IfStmt) from ret if the overload names matched
  678. # Only happens if no executable block had been found in IfStmt
  679. popped = ret.pop()
  680. assert isinstance(popped, IfStmt)
  681. skipped_if_stmts.append(popped)
  682. if current_overload and skipped_if_stmts:
  683. # Add bare IfStmt (without overloads) to ret
  684. # Required for mypy to be able to still check conditions
  685. for if_stmt in skipped_if_stmts:
  686. self._strip_contents_from_if_stmt(if_stmt)
  687. ret.append(if_stmt)
  688. skipped_if_stmts = []
  689. if len(current_overload) == 1:
  690. ret.append(current_overload[0])
  691. elif len(current_overload) > 1:
  692. ret.append(OverloadedFuncDef(current_overload))
  693. # If we have multiple decorated functions named "_" next to each, we want to treat
  694. # them as a series of regular FuncDefs instead of one OverloadedFuncDef because
  695. # most of mypy/mypyc assumes that all the functions in an OverloadedFuncDef are
  696. # related, but multiple underscore functions next to each other aren't necessarily
  697. # related
  698. seen_unconditional_func_def = False
  699. if isinstance(stmt, Decorator) and not unnamed_function(stmt.name):
  700. current_overload = [stmt]
  701. current_overload_name = stmt.name
  702. elif isinstance(stmt, IfStmt) and if_overload_name is not None:
  703. current_overload = []
  704. current_overload_name = if_overload_name
  705. last_if_stmt = stmt
  706. last_if_stmt_overload_name = None
  707. if if_block_with_overload is not None:
  708. skipped_if_stmts.extend(
  709. cast(List[IfStmt], if_block_with_overload.body[:-1])
  710. )
  711. last_if_overload = cast(
  712. Union[Decorator, FuncDef, OverloadedFuncDef],
  713. if_block_with_overload.body[-1],
  714. )
  715. last_if_unknown_truth_value = if_unknown_truth_value
  716. else:
  717. current_overload = []
  718. current_overload_name = None
  719. ret.append(stmt)
  720. if current_overload and skipped_if_stmts:
  721. # Add bare IfStmt (without overloads) to ret
  722. # Required for mypy to be able to still check conditions
  723. for if_stmt in skipped_if_stmts:
  724. self._strip_contents_from_if_stmt(if_stmt)
  725. ret.append(if_stmt)
  726. if len(current_overload) == 1:
  727. ret.append(current_overload[0])
  728. elif len(current_overload) > 1:
  729. ret.append(OverloadedFuncDef(current_overload))
  730. elif last_if_overload is not None:
  731. ret.append(last_if_overload)
  732. elif last_if_stmt is not None:
  733. ret.append(last_if_stmt)
  734. return ret
  735. def _check_ifstmt_for_overloads(
  736. self, stmt: IfStmt, current_overload_name: str | None = None
  737. ) -> str | None:
  738. """Check if IfStmt contains only overloads with the same name.
  739. Return overload_name if found, None otherwise.
  740. """
  741. # Check that block only contains a single Decorator, FuncDef, or OverloadedFuncDef.
  742. # Multiple overloads have already been merged as OverloadedFuncDef.
  743. if not (
  744. len(stmt.body[0].body) == 1
  745. and (
  746. isinstance(stmt.body[0].body[0], (Decorator, OverloadedFuncDef))
  747. or current_overload_name is not None
  748. and isinstance(stmt.body[0].body[0], FuncDef)
  749. )
  750. or len(stmt.body[0].body) > 1
  751. and isinstance(stmt.body[0].body[-1], OverloadedFuncDef)
  752. and all(self._is_stripped_if_stmt(if_stmt) for if_stmt in stmt.body[0].body[:-1])
  753. ):
  754. return None
  755. overload_name = cast(
  756. Union[Decorator, FuncDef, OverloadedFuncDef], stmt.body[0].body[-1]
  757. ).name
  758. if stmt.else_body is None:
  759. return overload_name
  760. if len(stmt.else_body.body) == 1:
  761. # For elif: else_body contains an IfStmt itself -> do a recursive check.
  762. if (
  763. isinstance(stmt.else_body.body[0], (Decorator, FuncDef, OverloadedFuncDef))
  764. and stmt.else_body.body[0].name == overload_name
  765. ):
  766. return overload_name
  767. if (
  768. isinstance(stmt.else_body.body[0], IfStmt)
  769. and self._check_ifstmt_for_overloads(stmt.else_body.body[0], current_overload_name)
  770. == overload_name
  771. ):
  772. return overload_name
  773. return None
  774. def _get_executable_if_block_with_overloads(
  775. self, stmt: IfStmt
  776. ) -> tuple[Block | None, IfStmt | None]:
  777. """Return block from IfStmt that will get executed.
  778. Return
  779. 0 -> A block if sure that alternative blocks are unreachable.
  780. 1 -> An IfStmt if the reachability of it can't be inferred,
  781. i.e. the truth value is unknown.
  782. """
  783. infer_reachability_of_if_statement(stmt, self.options)
  784. if stmt.else_body is None and stmt.body[0].is_unreachable is True:
  785. # always False condition with no else
  786. return None, None
  787. if (
  788. stmt.else_body is None
  789. or stmt.body[0].is_unreachable is False
  790. and stmt.else_body.is_unreachable is False
  791. ):
  792. # The truth value is unknown, thus not conclusive
  793. return None, stmt
  794. if stmt.else_body.is_unreachable is True:
  795. # else_body will be set unreachable if condition is always True
  796. return stmt.body[0], None
  797. if stmt.body[0].is_unreachable is True:
  798. # body will be set unreachable if condition is always False
  799. # else_body can contain an IfStmt itself (for elif) -> do a recursive check
  800. if isinstance(stmt.else_body.body[0], IfStmt):
  801. return self._get_executable_if_block_with_overloads(stmt.else_body.body[0])
  802. return stmt.else_body, None
  803. return None, stmt
  804. def _strip_contents_from_if_stmt(self, stmt: IfStmt) -> None:
  805. """Remove contents from IfStmt.
  806. Needed to still be able to check the conditions after the contents
  807. have been merged with the surrounding function overloads.
  808. """
  809. if len(stmt.body) == 1:
  810. stmt.body[0].body = []
  811. if stmt.else_body and len(stmt.else_body.body) == 1:
  812. if isinstance(stmt.else_body.body[0], IfStmt):
  813. self._strip_contents_from_if_stmt(stmt.else_body.body[0])
  814. else:
  815. stmt.else_body.body = []
  816. def _is_stripped_if_stmt(self, stmt: Statement) -> bool:
  817. """Check stmt to make sure it is a stripped IfStmt.
  818. See also: _strip_contents_from_if_stmt
  819. """
  820. if not isinstance(stmt, IfStmt):
  821. return False
  822. if not (len(stmt.body) == 1 and len(stmt.body[0].body) == 0):
  823. # Body not empty
  824. return False
  825. if not stmt.else_body or len(stmt.else_body.body) == 0:
  826. # No or empty else_body
  827. return True
  828. # For elif, IfStmt are stored recursively in else_body
  829. return self._is_stripped_if_stmt(stmt.else_body.body[0])
  830. def translate_module_id(self, id: str) -> str:
  831. """Return the actual, internal module id for a source text id."""
  832. if id == self.options.custom_typing_module:
  833. return "typing"
  834. return id
  835. def visit_Module(self, mod: ast3.Module) -> MypyFile:
  836. self.type_ignores = {}
  837. for ti in mod.type_ignores:
  838. parsed = parse_type_ignore_tag(ti.tag)
  839. if parsed is not None:
  840. self.type_ignores[ti.lineno] = parsed
  841. else:
  842. self.fail(message_registry.INVALID_TYPE_IGNORE, ti.lineno, -1, blocker=False)
  843. body = self.fix_function_overloads(self.translate_stmt_list(mod.body, ismodule=True))
  844. return MypyFile(body, self.imports, False, self.type_ignores)
  845. # --- stmt ---
  846. # FunctionDef(identifier name, arguments args,
  847. # stmt* body, expr* decorator_list, expr? returns, string? type_comment)
  848. # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
  849. # arg? kwarg, expr* defaults)
  850. def visit_FunctionDef(self, n: ast3.FunctionDef) -> FuncDef | Decorator:
  851. return self.do_func_def(n)
  852. # AsyncFunctionDef(identifier name, arguments args,
  853. # stmt* body, expr* decorator_list, expr? returns, string? type_comment)
  854. def visit_AsyncFunctionDef(self, n: ast3.AsyncFunctionDef) -> FuncDef | Decorator:
  855. return self.do_func_def(n, is_coroutine=True)
  856. def do_func_def(
  857. self, n: ast3.FunctionDef | ast3.AsyncFunctionDef, is_coroutine: bool = False
  858. ) -> FuncDef | Decorator:
  859. """Helper shared between visit_FunctionDef and visit_AsyncFunctionDef."""
  860. self.class_and_function_stack.append("D")
  861. no_type_check = bool(
  862. n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list)
  863. )
  864. lineno = n.lineno
  865. args = self.transform_args(n.args, lineno, no_type_check=no_type_check)
  866. if special_function_elide_names(n.name):
  867. for arg in args:
  868. arg.pos_only = True
  869. arg_kinds = [arg.kind for arg in args]
  870. arg_names = [None if arg.pos_only else arg.variable.name for arg in args]
  871. arg_types: list[Type | None] = []
  872. if no_type_check:
  873. arg_types = [None] * len(args)
  874. return_type = None
  875. elif n.type_comment is not None:
  876. try:
  877. func_type_ast = ast3_parse(n.type_comment, "<func_type>", "func_type")
  878. assert isinstance(func_type_ast, FunctionType)
  879. # for ellipsis arg
  880. if len(func_type_ast.argtypes) == 1 and isinstance(
  881. func_type_ast.argtypes[0], ast3_Ellipsis
  882. ):
  883. if n.returns:
  884. # PEP 484 disallows both type annotations and type comments
  885. self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset)
  886. arg_types = [
  887. a.type_annotation
  888. if a.type_annotation is not None
  889. else AnyType(TypeOfAny.unannotated)
  890. for a in args
  891. ]
  892. else:
  893. # PEP 484 disallows both type annotations and type comments
  894. if n.returns or any(a.type_annotation is not None for a in args):
  895. self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset)
  896. translated_args: list[Type] = TypeConverter(
  897. self.errors, line=lineno, override_column=n.col_offset
  898. ).translate_expr_list(func_type_ast.argtypes)
  899. # Use a cast to work around `list` invariance
  900. arg_types = cast(List[Optional[Type]], translated_args)
  901. return_type = TypeConverter(self.errors, line=lineno).visit(func_type_ast.returns)
  902. # add implicit self type
  903. in_method_scope = self.class_and_function_stack[-2:] == ["C", "D"]
  904. if in_method_scope and len(arg_types) < len(args):
  905. arg_types.insert(0, AnyType(TypeOfAny.special_form))
  906. except SyntaxError:
  907. stripped_type = n.type_comment.split("#", 2)[0].strip()
  908. err_msg = message_registry.TYPE_COMMENT_SYNTAX_ERROR_VALUE.format(stripped_type)
  909. self.fail(err_msg, lineno, n.col_offset)
  910. if n.type_comment and n.type_comment[0] not in ["(", "#"]:
  911. self.note(
  912. "Suggestion: wrap argument types in parentheses", lineno, n.col_offset
  913. )
  914. arg_types = [AnyType(TypeOfAny.from_error)] * len(args)
  915. return_type = AnyType(TypeOfAny.from_error)
  916. else:
  917. arg_types = [a.type_annotation for a in args]
  918. return_type = TypeConverter(
  919. self.errors, line=n.returns.lineno if n.returns else lineno
  920. ).visit(n.returns)
  921. for arg, arg_type in zip(args, arg_types):
  922. self.set_type_optional(arg_type, arg.initializer)
  923. func_type = None
  924. if any(arg_types) or return_type:
  925. if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types):
  926. self.fail(message_registry.ELLIPSIS_WITH_OTHER_TYPEARGS, lineno, n.col_offset)
  927. elif len(arg_types) > len(arg_kinds):
  928. self.fail(
  929. message_registry.TYPE_SIGNATURE_TOO_MANY_ARGS,
  930. lineno,
  931. n.col_offset,
  932. blocker=False,
  933. )
  934. elif len(arg_types) < len(arg_kinds):
  935. self.fail(
  936. message_registry.TYPE_SIGNATURE_TOO_FEW_ARGS,
  937. lineno,
  938. n.col_offset,
  939. blocker=False,
  940. )
  941. else:
  942. func_type = CallableType(
  943. [a if a is not None else AnyType(TypeOfAny.unannotated) for a in arg_types],
  944. arg_kinds,
  945. arg_names,
  946. return_type if return_type is not None else AnyType(TypeOfAny.unannotated),
  947. _dummy_fallback,
  948. )
  949. # End position is always the same.
  950. end_line = getattr(n, "end_lineno", None)
  951. end_column = getattr(n, "end_col_offset", None)
  952. self.class_and_function_stack.pop()
  953. self.class_and_function_stack.append("F")
  954. body = self.as_required_block(n.body, can_strip=True, is_coroutine=is_coroutine)
  955. func_def = FuncDef(n.name, args, body, func_type)
  956. if isinstance(func_def.type, CallableType):
  957. # semanal.py does some in-place modifications we want to avoid
  958. func_def.unanalyzed_type = func_def.type.copy_modified()
  959. if is_coroutine:
  960. func_def.is_coroutine = True
  961. if func_type is not None:
  962. func_type.definition = func_def
  963. func_type.line = lineno
  964. if n.decorator_list:
  965. if sys.version_info < (3, 8):
  966. # Before 3.8, [typed_]ast the line number points to the first decorator.
  967. # In 3.8, it points to the 'def' line, where we want it.
  968. deco_line = lineno
  969. lineno += len(n.decorator_list) # this is only approximately true
  970. else:
  971. # Set deco_line to the old pre-3.8 lineno, in order to keep
  972. # existing "# type: ignore" comments working:
  973. deco_line = n.decorator_list[0].lineno
  974. var = Var(func_def.name)
  975. var.is_ready = False
  976. var.set_line(lineno)
  977. func_def.is_decorated = True
  978. func_def.deco_line = deco_line
  979. func_def.set_line(lineno, n.col_offset, end_line, end_column)
  980. deco = Decorator(func_def, self.translate_expr_list(n.decorator_list), var)
  981. first = n.decorator_list[0]
  982. deco.set_line(first.lineno, first.col_offset, end_line, end_column)
  983. retval: FuncDef | Decorator = deco
  984. else:
  985. # FuncDef overrides set_line -- can't use self.set_line
  986. func_def.set_line(lineno, n.col_offset, end_line, end_column)
  987. retval = func_def
  988. self.class_and_function_stack.pop()
  989. return retval
  990. def set_type_optional(self, type: Type | None, initializer: Expression | None) -> None:
  991. if not self.options.implicit_optional:
  992. return
  993. # Indicate that type should be wrapped in an Optional if arg is initialized to None.
  994. optional = isinstance(initializer, NameExpr) and initializer.name == "None"
  995. if isinstance(type, UnboundType):
  996. type.optional = optional
  997. def transform_args(
  998. self, args: ast3.arguments, line: int, no_type_check: bool = False
  999. ) -> list[Argument]:
  1000. new_args = []
  1001. names: list[ast3.arg] = []
  1002. posonlyargs = getattr(args, "posonlyargs", cast(List[ast3.arg], []))
  1003. args_args = posonlyargs + args.args
  1004. args_defaults = args.defaults
  1005. num_no_defaults = len(args_args) - len(args_defaults)
  1006. # positional arguments without defaults
  1007. for i, a in enumerate(args_args[:num_no_defaults]):
  1008. pos_only = i < len(posonlyargs)
  1009. new_args.append(self.make_argument(a, None, ARG_POS, no_type_check, pos_only))
  1010. names.append(a)
  1011. # positional arguments with defaults
  1012. for i, (a, d) in enumerate(zip(args_args[num_no_defaults:], args_defaults)):
  1013. pos_only = num_no_defaults + i < len(posonlyargs)
  1014. new_args.append(self.make_argument(a, d, ARG_OPT, no_type_check, pos_only))
  1015. names.append(a)
  1016. # *arg
  1017. if args.vararg is not None:
  1018. new_args.append(self.make_argument(args.vararg, None, ARG_STAR, no_type_check))
  1019. names.append(args.vararg)
  1020. # keyword-only arguments with defaults
  1021. for a, kd in zip(args.kwonlyargs, args.kw_defaults):
  1022. new_args.append(
  1023. self.make_argument(
  1024. a, kd, ARG_NAMED if kd is None else ARG_NAMED_OPT, no_type_check
  1025. )
  1026. )
  1027. names.append(a)
  1028. # **kwarg
  1029. if args.kwarg is not None:
  1030. new_args.append(self.make_argument(args.kwarg, None, ARG_STAR2, no_type_check))
  1031. names.append(args.kwarg)
  1032. check_arg_names([arg.variable.name for arg in new_args], names, self.fail_arg)
  1033. return new_args
  1034. def make_argument(
  1035. self,
  1036. arg: ast3.arg,
  1037. default: ast3.expr | None,
  1038. kind: ArgKind,
  1039. no_type_check: bool,
  1040. pos_only: bool = False,
  1041. ) -> Argument:
  1042. if no_type_check:
  1043. arg_type = None
  1044. else:
  1045. annotation = arg.annotation
  1046. type_comment = arg.type_comment
  1047. if annotation is not None and type_comment is not None:
  1048. self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, arg.lineno, arg.col_offset)
  1049. arg_type = None
  1050. if annotation is not None:
  1051. arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation)
  1052. else:
  1053. arg_type = self.translate_type_comment(arg, type_comment)
  1054. if argument_elide_name(arg.arg):
  1055. pos_only = True
  1056. argument = Argument(Var(arg.arg), arg_type, self.visit(default), kind, pos_only)
  1057. argument.set_line(
  1058. arg.lineno,
  1059. arg.col_offset,
  1060. getattr(arg, "end_lineno", None),
  1061. getattr(arg, "end_col_offset", None),
  1062. )
  1063. return argument
  1064. def fail_arg(self, msg: str, arg: ast3.arg) -> None:
  1065. self.fail(ErrorMessage(msg), arg.lineno, arg.col_offset)
  1066. # ClassDef(identifier name,
  1067. # expr* bases,
  1068. # keyword* keywords,
  1069. # stmt* body,
  1070. # expr* decorator_list)
  1071. def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef:
  1072. self.class_and_function_stack.append("C")
  1073. keywords = [(kw.arg, self.visit(kw.value)) for kw in n.keywords if kw.arg]
  1074. cdef = ClassDef(
  1075. n.name,
  1076. self.as_required_block(n.body),
  1077. None,
  1078. self.translate_expr_list(n.bases),
  1079. metaclass=dict(keywords).get("metaclass"),
  1080. keywords=keywords,
  1081. )
  1082. cdef.decorators = self.translate_expr_list(n.decorator_list)
  1083. # Set lines to match the old mypy 0.700 lines, in order to keep
  1084. # existing "# type: ignore" comments working:
  1085. if sys.version_info < (3, 8):
  1086. cdef.line = n.lineno + len(n.decorator_list)
  1087. cdef.deco_line = n.lineno
  1088. else:
  1089. cdef.line = n.lineno
  1090. cdef.deco_line = n.decorator_list[0].lineno if n.decorator_list else None
  1091. cdef.column = n.col_offset
  1092. cdef.end_line = getattr(n, "end_lineno", None)
  1093. cdef.end_column = getattr(n, "end_col_offset", None)
  1094. self.class_and_function_stack.pop()
  1095. return cdef
  1096. # Return(expr? value)
  1097. def visit_Return(self, n: ast3.Return) -> ReturnStmt:
  1098. node = ReturnStmt(self.visit(n.value))
  1099. return self.set_line(node, n)
  1100. # Delete(expr* targets)
  1101. def visit_Delete(self, n: ast3.Delete) -> DelStmt:
  1102. if len(n.targets) > 1:
  1103. tup = TupleExpr(self.translate_expr_list(n.targets))
  1104. tup.set_line(n.lineno)
  1105. node = DelStmt(tup)
  1106. else:
  1107. node = DelStmt(self.visit(n.targets[0]))
  1108. return self.set_line(node, n)
  1109. # Assign(expr* targets, expr? value, string? type_comment, expr? annotation)
  1110. def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt:
  1111. lvalues = self.translate_expr_list(n.targets)
  1112. rvalue = self.visit(n.value)
  1113. typ = self.translate_type_comment(n, n.type_comment)
  1114. s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False)
  1115. return self.set_line(s, n)
  1116. # AnnAssign(expr target, expr annotation, expr? value, int simple)
  1117. def visit_AnnAssign(self, n: ast3.AnnAssign) -> AssignmentStmt:
  1118. line = n.lineno
  1119. if n.value is None: # always allow 'x: int'
  1120. rvalue: Expression = TempNode(AnyType(TypeOfAny.special_form), no_rhs=True)
  1121. rvalue.line = line
  1122. rvalue.column = n.col_offset
  1123. else:
  1124. rvalue = self.visit(n.value)
  1125. typ = TypeConverter(self.errors, line=line).visit(n.annotation)
  1126. assert typ is not None
  1127. typ.column = n.annotation.col_offset
  1128. s = AssignmentStmt([self.visit(n.target)], rvalue, type=typ, new_syntax=True)
  1129. return self.set_line(s, n)
  1130. # AugAssign(expr target, operator op, expr value)
  1131. def visit_AugAssign(self, n: ast3.AugAssign) -> OperatorAssignmentStmt:
  1132. s = OperatorAssignmentStmt(
  1133. self.from_operator(n.op), self.visit(n.target), self.visit(n.value)
  1134. )
  1135. return self.set_line(s, n)
  1136. # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
  1137. def visit_For(self, n: ast3.For) -> ForStmt:
  1138. target_type = self.translate_type_comment(n, n.type_comment)
  1139. node = ForStmt(
  1140. self.visit(n.target),
  1141. self.visit(n.iter),
  1142. self.as_required_block(n.body),
  1143. self.as_block(n.orelse),
  1144. target_type,
  1145. )
  1146. return self.set_line(node, n)
  1147. # AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
  1148. def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt:
  1149. target_type = self.translate_type_comment(n, n.type_comment)
  1150. node = ForStmt(
  1151. self.visit(n.target),
  1152. self.visit(n.iter),
  1153. self.as_required_block(n.body),
  1154. self.as_block(n.orelse),
  1155. target_type,
  1156. )
  1157. node.is_async = True
  1158. return self.set_line(node, n)
  1159. # While(expr test, stmt* body, stmt* orelse)
  1160. def visit_While(self, n: ast3.While) -> WhileStmt:
  1161. node = WhileStmt(
  1162. self.visit(n.test), self.as_required_block(n.body), self.as_block(n.orelse)
  1163. )
  1164. return self.set_line(node, n)
  1165. # If(expr test, stmt* body, stmt* orelse)
  1166. def visit_If(self, n: ast3.If) -> IfStmt:
  1167. node = IfStmt(
  1168. [self.visit(n.test)], [self.as_required_block(n.body)], self.as_block(n.orelse)
  1169. )
  1170. return self.set_line(node, n)
  1171. # With(withitem* items, stmt* body, string? type_comment)
  1172. def visit_With(self, n: ast3.With) -> WithStmt:
  1173. target_type = self.translate_type_comment(n, n.type_comment)
  1174. node = WithStmt(
  1175. [self.visit(i.context_expr) for i in n.items],
  1176. [self.visit(i.optional_vars) for i in n.items],
  1177. self.as_required_block(n.body),
  1178. target_type,
  1179. )
  1180. return self.set_line(node, n)
  1181. # AsyncWith(withitem* items, stmt* body, string? type_comment)
  1182. def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt:
  1183. target_type = self.translate_type_comment(n, n.type_comment)
  1184. s = WithStmt(
  1185. [self.visit(i.context_expr) for i in n.items],
  1186. [self.visit(i.optional_vars) for i in n.items],
  1187. self.as_required_block(n.body),
  1188. target_type,
  1189. )
  1190. s.is_async = True
  1191. return self.set_line(s, n)
  1192. # Raise(expr? exc, expr? cause)
  1193. def visit_Raise(self, n: ast3.Raise) -> RaiseStmt:
  1194. node = RaiseStmt(self.visit(n.exc), self.visit(n.cause))
  1195. return self.set_line(node, n)
  1196. # Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
  1197. def visit_Try(self, n: ast3.Try) -> TryStmt:
  1198. vs = [
  1199. self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers
  1200. ]
  1201. types = [self.visit(h.type) for h in n.handlers]
  1202. handlers = [self.as_required_block(h.body) for h in n.handlers]
  1203. node = TryStmt(
  1204. self.as_required_block(n.body),
  1205. vs,
  1206. types,
  1207. handlers,
  1208. self.as_block(n.orelse),
  1209. self.as_block(n.finalbody),
  1210. )
  1211. return self.set_line(node, n)
  1212. def visit_TryStar(self, n: TryStar) -> TryStmt:
  1213. vs = [
  1214. self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers
  1215. ]
  1216. types = [self.visit(h.type) for h in n.handlers]
  1217. handlers = [self.as_required_block(h.body) for h in n.handlers]
  1218. node = TryStmt(
  1219. self.as_required_block(n.body),
  1220. vs,
  1221. types,
  1222. handlers,
  1223. self.as_block(n.orelse),
  1224. self.as_block(n.finalbody),
  1225. )
  1226. node.is_star = True
  1227. return self.set_line(node, n)
  1228. # Assert(expr test, expr? msg)
  1229. def visit_Assert(self, n: ast3.Assert) -> AssertStmt:
  1230. node = AssertStmt(self.visit(n.test), self.visit(n.msg))
  1231. return self.set_line(node, n)
  1232. # Import(alias* names)
  1233. def visit_Import(self, n: ast3.Import) -> Import:
  1234. names: list[tuple[str, str | None]] = []
  1235. for alias in n.names:
  1236. name = self.translate_module_id(alias.name)
  1237. asname = alias.asname
  1238. if asname is None and name != alias.name:
  1239. # if the module name has been translated (and it's not already
  1240. # an explicit import-as), make it an implicit import-as the
  1241. # original name
  1242. asname = alias.name
  1243. names.append((name, asname))
  1244. i = Import(names)
  1245. self.imports.append(i)
  1246. return self.set_line(i, n)
  1247. # ImportFrom(identifier? module, alias* names, int? level)
  1248. def visit_ImportFrom(self, n: ast3.ImportFrom) -> ImportBase:
  1249. assert n.level is not None
  1250. if len(n.names) == 1 and n.names[0].name == "*":
  1251. mod = n.module if n.module is not None else ""
  1252. i: ImportBase = ImportAll(mod, n.level)
  1253. else:
  1254. i = ImportFrom(
  1255. self.translate_module_id(n.module) if n.module is not None else "",
  1256. n.level,
  1257. [(a.name, a.asname) for a in n.names],
  1258. )
  1259. self.imports.append(i)
  1260. return self.set_line(i, n)
  1261. # Global(identifier* names)
  1262. def visit_Global(self, n: ast3.Global) -> GlobalDecl:
  1263. g = GlobalDecl(n.names)
  1264. return self.set_line(g, n)
  1265. # Nonlocal(identifier* names)
  1266. def visit_Nonlocal(self, n: ast3.Nonlocal) -> NonlocalDecl:
  1267. d = NonlocalDecl(n.names)
  1268. return self.set_line(d, n)
  1269. # Expr(expr value)
  1270. def visit_Expr(self, n: ast3.Expr) -> ExpressionStmt:
  1271. value = self.visit(n.value)
  1272. node = ExpressionStmt(value)
  1273. return self.set_line(node, n)
  1274. # Pass
  1275. def visit_Pass(self, n: ast3.Pass) -> PassStmt:
  1276. s = PassStmt()
  1277. return self.set_line(s, n)
  1278. # Break
  1279. def visit_Break(self, n: ast3.Break) -> BreakStmt:
  1280. s = BreakStmt()
  1281. return self.set_line(s, n)
  1282. # Continue
  1283. def visit_Continue(self, n: ast3.Continue) -> ContinueStmt:
  1284. s = ContinueStmt()
  1285. return self.set_line(s, n)
  1286. # --- expr ---
  1287. def visit_NamedExpr(self, n: NamedExpr) -> AssignmentExpr:
  1288. s = AssignmentExpr(self.visit(n.target), self.visit(n.value))
  1289. return self.set_line(s, n)
  1290. # BoolOp(boolop op, expr* values)
  1291. def visit_BoolOp(self, n: ast3.BoolOp) -> OpExpr:
  1292. # mypy translates (1 and 2 and 3) as (1 and (2 and 3))
  1293. assert len(n.values) >= 2
  1294. op_node = n.op
  1295. if isinstance(op_node, ast3.And):
  1296. op = "and"
  1297. elif isinstance(op_node, ast3.Or):
  1298. op = "or"
  1299. else:
  1300. raise RuntimeError("unknown BoolOp " + str(type(n)))
  1301. # potentially inefficient!
  1302. return self.group(op, self.translate_expr_list(n.values), n)
  1303. def group(self, op: str, vals: list[Expression], n: ast3.expr) -> OpExpr:
  1304. if len(vals) == 2:
  1305. e = OpExpr(op, vals[0], vals[1])
  1306. else:
  1307. e = OpExpr(op, vals[0], self.group(op, vals[1:], n))
  1308. return self.set_line(e, n)
  1309. # BinOp(expr left, operator op, expr right)
  1310. def visit_BinOp(self, n: ast3.BinOp) -> OpExpr:
  1311. op = self.from_operator(n.op)
  1312. if op is None:
  1313. raise RuntimeError("cannot translate BinOp " + str(type(n.op)))
  1314. e = OpExpr(op, self.visit(n.left), self.visit(n.right))
  1315. return self.set_line(e, n)
  1316. # UnaryOp(unaryop op, expr operand)
  1317. def visit_UnaryOp(self, n: ast3.UnaryOp) -> UnaryExpr:
  1318. op = None
  1319. if isinstance(n.op, ast3.Invert):
  1320. op = "~"
  1321. elif isinstance(n.op, ast3.Not):
  1322. op = "not"
  1323. elif isinstance(n.op, ast3.UAdd):
  1324. op = "+"
  1325. elif isinstance(n.op, ast3.USub):
  1326. op = "-"
  1327. if op is None:
  1328. raise RuntimeError("cannot translate UnaryOp " + str(type(n.op)))
  1329. e = UnaryExpr(op, self.visit(n.operand))
  1330. return self.set_line(e, n)
  1331. # Lambda(arguments args, expr body)
  1332. def visit_Lambda(self, n: ast3.Lambda) -> LambdaExpr:
  1333. body = ast3.Return(n.body)
  1334. body.lineno = n.body.lineno
  1335. body.col_offset = n.body.col_offset
  1336. self.class_and_function_stack.append("L")
  1337. e = LambdaExpr(self.transform_args(n.args, n.lineno), self.as_required_block([body]))
  1338. self.class_and_function_stack.pop()
  1339. e.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line
  1340. return e
  1341. # IfExp(expr test, expr body, expr orelse)
  1342. def visit_IfExp(self, n: ast3.IfExp) -> ConditionalExpr:
  1343. e = ConditionalExpr(self.visit(n.test), self.visit(n.body), self.visit(n.orelse))
  1344. return self.set_line(e, n)
  1345. # Dict(expr* keys, expr* values)
  1346. def visit_Dict(self, n: ast3.Dict) -> DictExpr:
  1347. e = DictExpr(
  1348. list(zip(self.translate_opt_expr_list(n.keys), self.translate_expr_list(n.values)))
  1349. )
  1350. return self.set_line(e, n)
  1351. # Set(expr* elts)
  1352. def visit_Set(self, n: ast3.Set) -> SetExpr:
  1353. e = SetExpr(self.translate_expr_list(n.elts))
  1354. return self.set_line(e, n)
  1355. # ListComp(expr elt, comprehension* generators)
  1356. def visit_ListComp(self, n: ast3.ListComp) -> ListComprehension:
  1357. e = ListComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n)))
  1358. return self.set_line(e, n)
  1359. # SetComp(expr elt, comprehension* generators)
  1360. def visit_SetComp(self, n: ast3.SetComp) -> SetComprehension:
  1361. e = SetComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n)))
  1362. return self.set_line(e, n)
  1363. # DictComp(expr key, expr value, comprehension* generators)
  1364. def visit_DictComp(self, n: ast3.DictComp) -> DictionaryComprehension:
  1365. targets = [self.visit(c.target) for c in n.generators]
  1366. iters = [self.visit(c.iter) for c in n.generators]
  1367. ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators]
  1368. is_async = [bool(c.is_async) for c in n.generators]
  1369. e = DictionaryComprehension(
  1370. self.visit(n.key), self.visit(n.value), targets, iters, ifs_list, is_async
  1371. )
  1372. return self.set_line(e, n)
  1373. # GeneratorExp(expr elt, comprehension* generators)
  1374. def visit_GeneratorExp(self, n: ast3.GeneratorExp) -> GeneratorExpr:
  1375. targets = [self.visit(c.target) for c in n.generators]
  1376. iters = [self.visit(c.iter) for c in n.generators]
  1377. ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators]
  1378. is_async = [bool(c.is_async) for c in n.generators]
  1379. e = GeneratorExpr(self.visit(n.elt), targets, iters, ifs_list, is_async)
  1380. return self.set_line(e, n)
  1381. # Await(expr value)
  1382. def visit_Await(self, n: ast3.Await) -> AwaitExpr:
  1383. v = self.visit(n.value)
  1384. e = AwaitExpr(v)
  1385. return self.set_line(e, n)
  1386. # Yield(expr? value)
  1387. def visit_Yield(self, n: ast3.Yield) -> YieldExpr:
  1388. e = YieldExpr(self.visit(n.value))
  1389. return self.set_line(e, n)
  1390. # YieldFrom(expr value)
  1391. def visit_YieldFrom(self, n: ast3.YieldFrom) -> YieldFromExpr:
  1392. e = YieldFromExpr(self.visit(n.value))
  1393. return self.set_line(e, n)
  1394. # Compare(expr left, cmpop* ops, expr* comparators)
  1395. def visit_Compare(self, n: ast3.Compare) -> ComparisonExpr:
  1396. operators = [self.from_comp_operator(o) for o in n.ops]
  1397. operands = self.translate_expr_list([n.left] + n.comparators)
  1398. e = ComparisonExpr(operators, operands)
  1399. return self.set_line(e, n)
  1400. # Call(expr func, expr* args, keyword* keywords)
  1401. # keyword = (identifier? arg, expr value)
  1402. def visit_Call(self, n: Call) -> CallExpr:
  1403. args = n.args
  1404. keywords = n.keywords
  1405. keyword_names = [k.arg for k in keywords]
  1406. arg_types = self.translate_expr_list(
  1407. [a.value if isinstance(a, Starred) else a for a in args] + [k.value for k in keywords]
  1408. )
  1409. arg_kinds = [ARG_STAR if type(a) is Starred else ARG_POS for a in args] + [
  1410. ARG_STAR2 if arg is None else ARG_NAMED for arg in keyword_names
  1411. ]
  1412. e = CallExpr(
  1413. self.visit(n.func),
  1414. arg_types,
  1415. arg_kinds,
  1416. cast("List[Optional[str]]", [None] * len(args)) + keyword_names,
  1417. )
  1418. return self.set_line(e, n)
  1419. # Constant(object value) -- a constant, in Python 3.8.
  1420. def visit_Constant(self, n: Constant) -> Any:
  1421. val = n.value
  1422. e: Any = None
  1423. if val is None:
  1424. e = NameExpr("None")
  1425. elif isinstance(val, str):
  1426. e = StrExpr(n.s)
  1427. elif isinstance(val, bytes):
  1428. e = BytesExpr(bytes_to_human_readable_repr(n.s))
  1429. elif isinstance(val, bool): # Must check before int!
  1430. e = NameExpr(str(val))
  1431. elif isinstance(val, int):
  1432. e = IntExpr(val)
  1433. elif isinstance(val, float):
  1434. e = FloatExpr(val)
  1435. elif isinstance(val, complex):
  1436. e = ComplexExpr(val)
  1437. elif val is Ellipsis:
  1438. e = EllipsisExpr()
  1439. else:
  1440. raise RuntimeError("Constant not implemented for " + str(type(val)))
  1441. return self.set_line(e, n)
  1442. # Num(object n) -- a number as a PyObject.
  1443. def visit_Num(self, n: ast3.Num) -> IntExpr | FloatExpr | ComplexExpr:
  1444. # The n field has the type complex, but complex isn't *really*
  1445. # a parent of int and float, and this causes isinstance below
  1446. # to think that the complex branch is always picked. Avoid
  1447. # this by throwing away the type.
  1448. val: object = n.n
  1449. if isinstance(val, int):
  1450. e: IntExpr | FloatExpr | ComplexExpr = IntExpr(val)
  1451. elif isinstance(val, float):
  1452. e = FloatExpr(val)
  1453. elif isinstance(val, complex):
  1454. e = ComplexExpr(val)
  1455. else:
  1456. raise RuntimeError("num not implemented for " + str(type(val)))
  1457. return self.set_line(e, n)
  1458. # Str(string s)
  1459. def visit_Str(self, n: Str) -> StrExpr:
  1460. e = StrExpr(n.s)
  1461. return self.set_line(e, n)
  1462. # JoinedStr(expr* values)
  1463. def visit_JoinedStr(self, n: ast3.JoinedStr) -> Expression:
  1464. # Each of n.values is a str or FormattedValue; we just concatenate
  1465. # them all using ''.join.
  1466. empty_string = StrExpr("")
  1467. empty_string.set_line(n.lineno, n.col_offset)
  1468. strs_to_join = ListExpr(self.translate_expr_list(n.values))
  1469. strs_to_join.set_line(empty_string)
  1470. # Don't make unnecessary join call if there is only one str to join
  1471. if len(strs_to_join.items) == 1:
  1472. return self.set_line(strs_to_join.items[0], n)
  1473. join_method = MemberExpr(empty_string, "join")
  1474. join_method.set_line(empty_string)
  1475. result_expression = CallExpr(join_method, [strs_to_join], [ARG_POS], [None])
  1476. return self.set_line(result_expression, n)
  1477. # FormattedValue(expr value)
  1478. def visit_FormattedValue(self, n: ast3.FormattedValue) -> Expression:
  1479. # A FormattedValue is a component of a JoinedStr, or it can exist
  1480. # on its own. We translate them to individual '{}'.format(value)
  1481. # calls. Format specifier and conversion information is passed along
  1482. # to allow mypyc to support f-strings with format specifiers and conversions.
  1483. val_exp = self.visit(n.value)
  1484. val_exp.set_line(n.lineno, n.col_offset)
  1485. conv_str = "" if n.conversion is None or n.conversion < 0 else "!" + chr(n.conversion)
  1486. format_string = StrExpr("{" + conv_str + ":{}}")
  1487. format_spec_exp = self.visit(n.format_spec) if n.format_spec is not None else StrExpr("")
  1488. format_string.set_line(n.lineno, n.col_offset)
  1489. format_method = MemberExpr(format_string, "format")
  1490. format_method.set_line(format_string)
  1491. result_expression = CallExpr(
  1492. format_method, [val_exp, format_spec_exp], [ARG_POS, ARG_POS], [None, None]
  1493. )
  1494. return self.set_line(result_expression, n)
  1495. # Bytes(bytes s)
  1496. def visit_Bytes(self, n: ast3.Bytes) -> BytesExpr | StrExpr:
  1497. e = BytesExpr(bytes_to_human_readable_repr(n.s))
  1498. return self.set_line(e, n)
  1499. # NameConstant(singleton value)
  1500. def visit_NameConstant(self, n: NameConstant) -> NameExpr:
  1501. e = NameExpr(str(n.value))
  1502. return self.set_line(e, n)
  1503. # Ellipsis
  1504. def visit_Ellipsis(self, n: ast3_Ellipsis) -> EllipsisExpr:
  1505. e = EllipsisExpr()
  1506. return self.set_line(e, n)
  1507. # Attribute(expr value, identifier attr, expr_context ctx)
  1508. def visit_Attribute(self, n: Attribute) -> MemberExpr | SuperExpr:
  1509. value = n.value
  1510. member_expr = MemberExpr(self.visit(value), n.attr)
  1511. obj = member_expr.expr
  1512. if (
  1513. isinstance(obj, CallExpr)
  1514. and isinstance(obj.callee, NameExpr)
  1515. and obj.callee.name == "super"
  1516. ):
  1517. e: MemberExpr | SuperExpr = SuperExpr(member_expr.name, obj)
  1518. else:
  1519. e = member_expr
  1520. return self.set_line(e, n)
  1521. # Subscript(expr value, slice slice, expr_context ctx)
  1522. def visit_Subscript(self, n: ast3.Subscript) -> IndexExpr:
  1523. e = IndexExpr(self.visit(n.value), self.visit(n.slice))
  1524. self.set_line(e, n)
  1525. # alias to please mypyc
  1526. is_py38_or_earlier = sys.version_info < (3, 9)
  1527. if isinstance(n.slice, ast3.Slice) or (
  1528. is_py38_or_earlier and isinstance(n.slice, ast3.ExtSlice)
  1529. ):
  1530. # Before Python 3.9, Slice has no line/column in the raw ast. To avoid incompatibility
  1531. # visit_Slice doesn't set_line, even in Python 3.9 on.
  1532. # ExtSlice also has no line/column info. In Python 3.9 on, line/column is set for
  1533. # e.index when visiting n.slice.
  1534. e.index.line = e.line
  1535. e.index.column = e.column
  1536. return e
  1537. # Starred(expr value, expr_context ctx)
  1538. def visit_Starred(self, n: Starred) -> StarExpr:
  1539. e = StarExpr(self.visit(n.value))
  1540. return self.set_line(e, n)
  1541. # Name(identifier id, expr_context ctx)
  1542. def visit_Name(self, n: Name) -> NameExpr:
  1543. e = NameExpr(n.id)
  1544. return self.set_line(e, n)
  1545. # List(expr* elts, expr_context ctx)
  1546. def visit_List(self, n: ast3.List) -> ListExpr | TupleExpr:
  1547. expr_list: list[Expression] = [self.visit(e) for e in n.elts]
  1548. if isinstance(n.ctx, ast3.Store):
  1549. # [x, y] = z and (x, y) = z means exactly the same thing
  1550. e: ListExpr | TupleExpr = TupleExpr(expr_list)
  1551. else:
  1552. e = ListExpr(expr_list)
  1553. return self.set_line(e, n)
  1554. # Tuple(expr* elts, expr_context ctx)
  1555. def visit_Tuple(self, n: ast3.Tuple) -> TupleExpr:
  1556. e = TupleExpr(self.translate_expr_list(n.elts))
  1557. return self.set_line(e, n)
  1558. # --- slice ---
  1559. # Slice(expr? lower, expr? upper, expr? step)
  1560. def visit_Slice(self, n: ast3.Slice) -> SliceExpr:
  1561. return SliceExpr(self.visit(n.lower), self.visit(n.upper), self.visit(n.step))
  1562. # ExtSlice(slice* dims)
  1563. def visit_ExtSlice(self, n: ast3.ExtSlice) -> TupleExpr:
  1564. # cast for mypyc's benefit on Python 3.9
  1565. return TupleExpr(self.translate_expr_list(cast(Any, n).dims))
  1566. # Index(expr value)
  1567. def visit_Index(self, n: Index) -> Node:
  1568. # cast for mypyc's benefit on Python 3.9
  1569. value = self.visit(cast(Any, n).value)
  1570. assert isinstance(value, Node)
  1571. return value
  1572. # Match(expr subject, match_case* cases) # python 3.10 and later
  1573. def visit_Match(self, n: Match) -> MatchStmt:
  1574. node = MatchStmt(
  1575. self.visit(n.subject),
  1576. [self.visit(c.pattern) for c in n.cases],
  1577. [self.visit(c.guard) for c in n.cases],
  1578. [self.as_required_block(c.body) for c in n.cases],
  1579. )
  1580. return self.set_line(node, n)
  1581. def visit_MatchValue(self, n: MatchValue) -> ValuePattern:
  1582. node = ValuePattern(self.visit(n.value))
  1583. return self.set_line(node, n)
  1584. def visit_MatchSingleton(self, n: MatchSingleton) -> SingletonPattern:
  1585. node = SingletonPattern(n.value)
  1586. return self.set_line(node, n)
  1587. def visit_MatchSequence(self, n: MatchSequence) -> SequencePattern:
  1588. patterns = [self.visit(p) for p in n.patterns]
  1589. stars = [p for p in patterns if isinstance(p, StarredPattern)]
  1590. assert len(stars) < 2
  1591. node = SequencePattern(patterns)
  1592. return self.set_line(node, n)
  1593. def visit_MatchStar(self, n: MatchStar) -> StarredPattern:
  1594. if n.name is None:
  1595. node = StarredPattern(None)
  1596. else:
  1597. name = self.set_line(NameExpr(n.name), n)
  1598. node = StarredPattern(name)
  1599. return self.set_line(node, n)
  1600. def visit_MatchMapping(self, n: MatchMapping) -> MappingPattern:
  1601. keys = [self.visit(k) for k in n.keys]
  1602. values = [self.visit(v) for v in n.patterns]
  1603. if n.rest is None:
  1604. rest = None
  1605. else:
  1606. rest = NameExpr(n.rest)
  1607. node = MappingPattern(keys, values, rest)
  1608. return self.set_line(node, n)
  1609. def visit_MatchClass(self, n: MatchClass) -> ClassPattern:
  1610. class_ref = self.visit(n.cls)
  1611. assert isinstance(class_ref, RefExpr)
  1612. positionals = [self.visit(p) for p in n.patterns]
  1613. keyword_keys = n.kwd_attrs
  1614. keyword_values = [self.visit(p) for p in n.kwd_patterns]
  1615. node = ClassPattern(class_ref, positionals, keyword_keys, keyword_values)
  1616. return self.set_line(node, n)
  1617. # MatchAs(expr pattern, identifier name)
  1618. def visit_MatchAs(self, n: MatchAs) -> AsPattern:
  1619. if n.name is None:
  1620. name = None
  1621. else:
  1622. name = NameExpr(n.name)
  1623. name = self.set_line(name, n)
  1624. node = AsPattern(self.visit(n.pattern), name)
  1625. return self.set_line(node, n)
  1626. # MatchOr(expr* pattern)
  1627. def visit_MatchOr(self, n: MatchOr) -> OrPattern:
  1628. node = OrPattern([self.visit(pattern) for pattern in n.patterns])
  1629. return self.set_line(node, n)
  1630. class TypeConverter:
  1631. def __init__(
  1632. self,
  1633. errors: Errors | None,
  1634. line: int = -1,
  1635. override_column: int = -1,
  1636. is_evaluated: bool = True,
  1637. ) -> None:
  1638. self.errors = errors
  1639. self.line = line
  1640. self.override_column = override_column
  1641. self.node_stack: list[AST] = []
  1642. self.is_evaluated = is_evaluated
  1643. def convert_column(self, column: int) -> int:
  1644. """Apply column override if defined; otherwise return column.
  1645. Column numbers are sometimes incorrect in the AST and the column
  1646. override can be used to work around that.
  1647. """
  1648. if self.override_column < 0:
  1649. return column
  1650. else:
  1651. return self.override_column
  1652. def invalid_type(self, node: AST, note: str | None = None) -> RawExpressionType:
  1653. """Constructs a type representing some expression that normally forms an invalid type.
  1654. For example, if we see a type hint that says "3 + 4", we would transform that
  1655. expression into a RawExpressionType.
  1656. The semantic analysis layer will report an "Invalid type" error when it
  1657. encounters this type, along with the given note if one is provided.
  1658. See RawExpressionType's docstring for more details on how it's used.
  1659. """
  1660. return RawExpressionType(
  1661. None, "typing.Any", line=self.line, column=getattr(node, "col_offset", -1), note=note
  1662. )
  1663. @overload
  1664. def visit(self, node: ast3.expr) -> ProperType:
  1665. ...
  1666. @overload
  1667. def visit(self, node: AST | None) -> ProperType | None:
  1668. ...
  1669. def visit(self, node: AST | None) -> ProperType | None:
  1670. """Modified visit -- keep track of the stack of nodes"""
  1671. if node is None:
  1672. return None
  1673. self.node_stack.append(node)
  1674. try:
  1675. method = "visit_" + node.__class__.__name__
  1676. visitor = getattr(self, method, None)
  1677. if visitor is not None:
  1678. typ = visitor(node)
  1679. assert isinstance(typ, ProperType)
  1680. return typ
  1681. else:
  1682. return self.invalid_type(node)
  1683. finally:
  1684. self.node_stack.pop()
  1685. def parent(self) -> AST | None:
  1686. """Return the AST node above the one we are processing"""
  1687. if len(self.node_stack) < 2:
  1688. return None
  1689. return self.node_stack[-2]
  1690. def fail(self, msg: ErrorMessage, line: int, column: int) -> None:
  1691. if self.errors:
  1692. self.errors.report(line, column, msg.value, blocker=True, code=msg.code)
  1693. def note(self, msg: str, line: int, column: int) -> None:
  1694. if self.errors:
  1695. self.errors.report(line, column, msg, severity="note", code=codes.SYNTAX)
  1696. def translate_expr_list(self, l: Sequence[ast3.expr]) -> list[Type]:
  1697. return [self.visit(e) for e in l]
  1698. def visit_Call(self, e: Call) -> Type:
  1699. # Parse the arg constructor
  1700. f = e.func
  1701. constructor = stringify_name(f)
  1702. if not isinstance(self.parent(), ast3.List):
  1703. note = None
  1704. if constructor:
  1705. note = "Suggestion: use {0}[...] instead of {0}(...)".format(constructor)
  1706. return self.invalid_type(e, note=note)
  1707. if not constructor:
  1708. self.fail(message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED, e.lineno, e.col_offset)
  1709. name: str | None = None
  1710. default_type = AnyType(TypeOfAny.special_form)
  1711. typ: Type = default_type
  1712. for i, arg in enumerate(e.args):
  1713. if i == 0:
  1714. converted = self.visit(arg)
  1715. assert converted is not None
  1716. typ = converted
  1717. elif i == 1:
  1718. name = self._extract_argument_name(arg)
  1719. else:
  1720. self.fail(message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS, f.lineno, f.col_offset)
  1721. for k in e.keywords:
  1722. value = k.value
  1723. if k.arg == "name":
  1724. if name is not None:
  1725. self.fail(
  1726. message_registry.MULTIPLE_VALUES_FOR_NAME_KWARG.format(constructor),
  1727. f.lineno,
  1728. f.col_offset,
  1729. )
  1730. name = self._extract_argument_name(value)
  1731. elif k.arg == "type":
  1732. if typ is not default_type:
  1733. self.fail(
  1734. message_registry.MULTIPLE_VALUES_FOR_TYPE_KWARG.format(constructor),
  1735. f.lineno,
  1736. f.col_offset,
  1737. )
  1738. converted = self.visit(value)
  1739. assert converted is not None
  1740. typ = converted
  1741. else:
  1742. self.fail(
  1743. message_registry.ARG_CONSTRUCTOR_UNEXPECTED_ARG.format(k.arg),
  1744. value.lineno,
  1745. value.col_offset,
  1746. )
  1747. return CallableArgument(typ, name, constructor, e.lineno, e.col_offset)
  1748. def translate_argument_list(self, l: Sequence[ast3.expr]) -> TypeList:
  1749. return TypeList([self.visit(e) for e in l], line=self.line)
  1750. def _extract_argument_name(self, n: ast3.expr) -> str | None:
  1751. if isinstance(n, Str):
  1752. return n.s.strip()
  1753. elif isinstance(n, NameConstant) and str(n.value) == "None":
  1754. return None
  1755. self.fail(
  1756. message_registry.ARG_NAME_EXPECTED_STRING_LITERAL.format(type(n).__name__),
  1757. self.line,
  1758. 0,
  1759. )
  1760. return None
  1761. def visit_Name(self, n: Name) -> Type:
  1762. return UnboundType(n.id, line=self.line, column=self.convert_column(n.col_offset))
  1763. def visit_BinOp(self, n: ast3.BinOp) -> Type:
  1764. if not isinstance(n.op, ast3.BitOr):
  1765. return self.invalid_type(n)
  1766. left = self.visit(n.left)
  1767. right = self.visit(n.right)
  1768. return UnionType(
  1769. [left, right],
  1770. line=self.line,
  1771. column=self.convert_column(n.col_offset),
  1772. is_evaluated=self.is_evaluated,
  1773. uses_pep604_syntax=True,
  1774. )
  1775. def visit_NameConstant(self, n: NameConstant) -> Type:
  1776. if isinstance(n.value, bool):
  1777. return RawExpressionType(n.value, "builtins.bool", line=self.line)
  1778. else:
  1779. return UnboundType(str(n.value), line=self.line, column=n.col_offset)
  1780. # Only for 3.8 and newer
  1781. def visit_Constant(self, n: Constant) -> Type:
  1782. val = n.value
  1783. if val is None:
  1784. # None is a type.
  1785. return UnboundType("None", line=self.line)
  1786. if isinstance(val, str):
  1787. # Parse forward reference.
  1788. return parse_type_string(n.s, "builtins.str", self.line, n.col_offset)
  1789. if val is Ellipsis:
  1790. # '...' is valid in some types.
  1791. return EllipsisType(line=self.line)
  1792. if isinstance(val, bool):
  1793. # Special case for True/False.
  1794. return RawExpressionType(val, "builtins.bool", line=self.line)
  1795. if isinstance(val, (int, float, complex)):
  1796. return self.numeric_type(val, n)
  1797. if isinstance(val, bytes):
  1798. contents = bytes_to_human_readable_repr(val)
  1799. return RawExpressionType(contents, "builtins.bytes", self.line, column=n.col_offset)
  1800. # Everything else is invalid.
  1801. return self.invalid_type(n)
  1802. # UnaryOp(op, operand)
  1803. def visit_UnaryOp(self, n: UnaryOp) -> Type:
  1804. # We support specifically Literal[-4] and nothing else.
  1805. # For example, Literal[+4] or Literal[~6] is not supported.
  1806. typ = self.visit(n.operand)
  1807. if isinstance(typ, RawExpressionType) and isinstance(n.op, USub):
  1808. if isinstance(typ.literal_value, int):
  1809. typ.literal_value *= -1
  1810. return typ
  1811. return self.invalid_type(n)
  1812. def numeric_type(self, value: object, n: AST) -> Type:
  1813. # The node's field has the type complex, but complex isn't *really*
  1814. # a parent of int and float, and this causes isinstance below
  1815. # to think that the complex branch is always picked. Avoid
  1816. # this by throwing away the type.
  1817. if isinstance(value, int):
  1818. numeric_value: int | None = value
  1819. type_name = "builtins.int"
  1820. else:
  1821. # Other kinds of numbers (floats, complex) are not valid parameters for
  1822. # RawExpressionType so we just pass in 'None' for now. We'll report the
  1823. # appropriate error at a later stage.
  1824. numeric_value = None
  1825. type_name = f"builtins.{type(value).__name__}"
  1826. return RawExpressionType(
  1827. numeric_value, type_name, line=self.line, column=getattr(n, "col_offset", -1)
  1828. )
  1829. # These next three methods are only used if we are on python <
  1830. # 3.8, using typed_ast. They are defined unconditionally because
  1831. # mypyc can't handle conditional method definitions.
  1832. # Num(number n)
  1833. def visit_Num(self, n: Num) -> Type:
  1834. return self.numeric_type(n.n, n)
  1835. # Str(string s)
  1836. def visit_Str(self, n: Str) -> Type:
  1837. return parse_type_string(n.s, "builtins.str", self.line, n.col_offset)
  1838. # Bytes(bytes s)
  1839. def visit_Bytes(self, n: Bytes) -> Type:
  1840. contents = bytes_to_human_readable_repr(n.s)
  1841. return RawExpressionType(contents, "builtins.bytes", self.line, column=n.col_offset)
  1842. def visit_Index(self, n: ast3.Index) -> Type:
  1843. # cast for mypyc's benefit on Python 3.9
  1844. value = self.visit(cast(Any, n).value)
  1845. assert isinstance(value, Type)
  1846. return value
  1847. def visit_Slice(self, n: ast3.Slice) -> Type:
  1848. return self.invalid_type(n, note="did you mean to use ',' instead of ':' ?")
  1849. # Subscript(expr value, slice slice, expr_context ctx) # Python 3.8 and before
  1850. # Subscript(expr value, expr slice, expr_context ctx) # Python 3.9 and later
  1851. def visit_Subscript(self, n: ast3.Subscript) -> Type:
  1852. if sys.version_info >= (3, 9): # Really 3.9a5 or later
  1853. sliceval: Any = n.slice
  1854. # Python 3.8 or earlier use a different AST structure for subscripts
  1855. elif isinstance(n.slice, ast3.Index):
  1856. sliceval: Any = n.slice.value
  1857. elif isinstance(n.slice, ast3.Slice):
  1858. sliceval = copy.deepcopy(n.slice) # so we don't mutate passed AST
  1859. if getattr(sliceval, "col_offset", None) is None:
  1860. # Fix column information so that we get Python 3.9+ message order
  1861. sliceval.col_offset = sliceval.lower.col_offset
  1862. else:
  1863. assert isinstance(n.slice, ast3.ExtSlice)
  1864. dims = copy.deepcopy(n.slice.dims)
  1865. for s in dims:
  1866. if getattr(s, "col_offset", None) is None:
  1867. if isinstance(s, ast3.Index):
  1868. s.col_offset = s.value.col_offset # type: ignore[attr-defined]
  1869. elif isinstance(s, ast3.Slice):
  1870. assert s.lower is not None
  1871. s.col_offset = s.lower.col_offset # type: ignore[attr-defined]
  1872. sliceval = ast3.Tuple(dims, n.ctx)
  1873. empty_tuple_index = False
  1874. if isinstance(sliceval, ast3.Tuple):
  1875. params = self.translate_expr_list(sliceval.elts)
  1876. if len(sliceval.elts) == 0:
  1877. empty_tuple_index = True
  1878. else:
  1879. params = [self.visit(sliceval)]
  1880. value = self.visit(n.value)
  1881. if isinstance(value, UnboundType) and not value.args:
  1882. return UnboundType(
  1883. value.name,
  1884. params,
  1885. line=self.line,
  1886. column=value.column,
  1887. empty_tuple_index=empty_tuple_index,
  1888. )
  1889. else:
  1890. return self.invalid_type(n)
  1891. def visit_Tuple(self, n: ast3.Tuple) -> Type:
  1892. return TupleType(
  1893. self.translate_expr_list(n.elts),
  1894. _dummy_fallback,
  1895. implicit=True,
  1896. line=self.line,
  1897. column=self.convert_column(n.col_offset),
  1898. )
  1899. # Attribute(expr value, identifier attr, expr_context ctx)
  1900. def visit_Attribute(self, n: Attribute) -> Type:
  1901. before_dot = self.visit(n.value)
  1902. if isinstance(before_dot, UnboundType) and not before_dot.args:
  1903. return UnboundType(f"{before_dot.name}.{n.attr}", line=self.line)
  1904. else:
  1905. return self.invalid_type(n)
  1906. # Ellipsis
  1907. def visit_Ellipsis(self, n: ast3_Ellipsis) -> Type:
  1908. return EllipsisType(line=self.line)
  1909. # List(expr* elts, expr_context ctx)
  1910. def visit_List(self, n: ast3.List) -> Type:
  1911. assert isinstance(n.ctx, ast3.Load)
  1912. return self.translate_argument_list(n.elts)
  1913. def stringify_name(n: AST) -> str | None:
  1914. if isinstance(n, Name):
  1915. return n.id
  1916. elif isinstance(n, Attribute):
  1917. sv = stringify_name(n.value)
  1918. if sv is not None:
  1919. return f"{sv}.{n.attr}"
  1920. return None # Can't do it.
  1921. class FindAttributeAssign(TraverserVisitor):
  1922. """Check if an AST contains attribute assignments (e.g. self.x = 0)."""
  1923. def __init__(self) -> None:
  1924. self.lvalue = False
  1925. self.found = False
  1926. def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
  1927. self.lvalue = True
  1928. for lv in s.lvalues:
  1929. lv.accept(self)
  1930. self.lvalue = False
  1931. def visit_with_stmt(self, s: WithStmt) -> None:
  1932. self.lvalue = True
  1933. for lv in s.target:
  1934. if lv is not None:
  1935. lv.accept(self)
  1936. self.lvalue = False
  1937. s.body.accept(self)
  1938. def visit_for_stmt(self, s: ForStmt) -> None:
  1939. self.lvalue = True
  1940. s.index.accept(self)
  1941. self.lvalue = False
  1942. s.body.accept(self)
  1943. if s.else_body:
  1944. s.else_body.accept(self)
  1945. def visit_expression_stmt(self, s: ExpressionStmt) -> None:
  1946. # No need to look inside these
  1947. pass
  1948. def visit_call_expr(self, e: CallExpr) -> None:
  1949. # No need to look inside these
  1950. pass
  1951. def visit_index_expr(self, e: IndexExpr) -> None:
  1952. # No need to look inside these
  1953. pass
  1954. def visit_member_expr(self, e: MemberExpr) -> None:
  1955. if self.lvalue:
  1956. self.found = True
  1957. class FindYield(TraverserVisitor):
  1958. """Check if an AST contains yields or yield froms."""
  1959. def __init__(self) -> None:
  1960. self.found = False
  1961. def visit_yield_expr(self, e: YieldExpr) -> None:
  1962. self.found = True
  1963. def visit_yield_from_expr(self, e: YieldFromExpr) -> None:
  1964. self.found = True
  1965. def is_possible_trivial_body(s: list[Statement]) -> bool:
  1966. """Could the statements form a "trivial" function body, such as 'pass'?
  1967. This mimics mypy.semanal.is_trivial_body, but this runs before
  1968. semantic analysis so some checks must be conservative.
  1969. """
  1970. l = len(s)
  1971. if l == 0:
  1972. return False
  1973. i = 0
  1974. if isinstance(s[0], ExpressionStmt) and isinstance(s[0].expr, StrExpr):
  1975. # Skip docstring
  1976. i += 1
  1977. if i == l:
  1978. return True
  1979. if l > i + 1:
  1980. return False
  1981. stmt = s[i]
  1982. return isinstance(stmt, (PassStmt, RaiseStmt)) or (
  1983. isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr)
  1984. )