testupdatedata.py 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154
  1. import shlex
  2. import subprocess
  3. import sys
  4. import textwrap
  5. from pathlib import Path
  6. from mypy.test.config import test_data_prefix
  7. from mypy.test.helpers import Suite
  8. class UpdateDataSuite(Suite):
  9. def _run_pytest_update_data(self, data_suite: str, *, max_attempts: int) -> str:
  10. """
  11. Runs a suite of data test cases through 'pytest --update-data' until either tests pass
  12. or until a maximum number of attempts (needed for incremental tests).
  13. """
  14. p_test_data = Path(test_data_prefix)
  15. p_root = p_test_data.parent.parent
  16. p = p_test_data / "check-update-data.test"
  17. assert not p.exists()
  18. try:
  19. p.write_text(textwrap.dedent(data_suite).lstrip())
  20. test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}"
  21. args = [sys.executable, "-m", "pytest", "-n", "0", "-s", "--update-data", test_nodeid]
  22. if sys.version_info >= (3, 8):
  23. cmd = shlex.join(args)
  24. else:
  25. cmd = " ".join(args)
  26. for i in range(max_attempts - 1, -1, -1):
  27. res = subprocess.run(args, cwd=p_root)
  28. if res.returncode == 0:
  29. break
  30. print(f"`{cmd}` returned {res.returncode}: {i} attempts remaining")
  31. return p.read_text()
  32. finally:
  33. p.unlink()
  34. def test_update_data(self) -> None:
  35. # Note: We test multiple testcases rather than 'test case per test case'
  36. # so we could also exercise rewriting multiple testcases at once.
  37. actual = self._run_pytest_update_data(
  38. """
  39. [case testCorrect]
  40. s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
  41. [case testWrong]
  42. s: str = 42 # E: wrong error
  43. [case testXfail-xfail]
  44. s: str = 42 # E: wrong error
  45. [case testWrongMultiline]
  46. s: str = 42 # E: foo \
  47. # N: bar
  48. [case testMissingMultiline]
  49. s: str = 42; i: int = 'foo'
  50. [case testExtraneous]
  51. s: str = 'foo' # E: wrong error
  52. [case testExtraneousMultiline]
  53. s: str = 'foo' # E: foo \
  54. # E: bar
  55. [case testExtraneousMultilineNonError]
  56. s: str = 'foo' # W: foo \
  57. # N: bar
  58. [case testOutCorrect]
  59. s: str = 42
  60. [out]
  61. main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
  62. [case testOutWrong]
  63. s: str = 42
  64. [out]
  65. main:1: error: foobar
  66. [case testOutWrongIncremental]
  67. s: str = 42
  68. [out]
  69. main:1: error: foobar
  70. [out2]
  71. main:1: error: foobar
  72. [case testWrongMultipleFiles]
  73. import a, b
  74. s: str = 42 # E: foo
  75. [file a.py]
  76. s1: str = 42 # E: bar
  77. [file b.py]
  78. s2: str = 43 # E: baz
  79. [builtins fixtures/list.pyi]
  80. """,
  81. max_attempts=3,
  82. )
  83. # Assert
  84. expected = """
  85. [case testCorrect]
  86. s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
  87. [case testWrong]
  88. s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
  89. [case testXfail-xfail]
  90. s: str = 42 # E: wrong error
  91. [case testWrongMultiline]
  92. s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
  93. [case testMissingMultiline]
  94. s: str = 42; i: int = 'foo' # E: Incompatible types in assignment (expression has type "int", variable has type "str") \\
  95. # E: Incompatible types in assignment (expression has type "str", variable has type "int")
  96. [case testExtraneous]
  97. s: str = 'foo'
  98. [case testExtraneousMultiline]
  99. s: str = 'foo'
  100. [case testExtraneousMultilineNonError]
  101. s: str = 'foo'
  102. [case testOutCorrect]
  103. s: str = 42
  104. [out]
  105. main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
  106. [case testOutWrong]
  107. s: str = 42
  108. [out]
  109. main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
  110. [case testOutWrongIncremental]
  111. s: str = 42
  112. [out]
  113. main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
  114. [out2]
  115. main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
  116. [case testWrongMultipleFiles]
  117. import a, b
  118. s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
  119. [file a.py]
  120. s1: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
  121. [file b.py]
  122. s2: str = 43 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
  123. [builtins fixtures/list.pyi]
  124. """
  125. assert actual == textwrap.dedent(expected).lstrip()