158 lines
5.5 KiB
Python
158 lines
5.5 KiB
Python
|
"""
|
||
|
A "meta test" which tests the `--update-data` feature for updating .test files.
|
||
|
Updating the expected output, especially when it's in the form of inline (comment) assertions,
|
||
|
can be brittle, which is why we're "meta-testing" here.
|
||
|
"""
|
||
|
import shlex
|
||
|
import subprocess
|
||
|
import sys
|
||
|
import textwrap
|
||
|
import uuid
|
||
|
from pathlib import Path
|
||
|
|
||
|
from mypy.test.config import test_data_prefix
|
||
|
from mypy.test.helpers import Suite
|
||
|
|
||
|
|
||
|
class UpdateDataSuite(Suite):
|
||
|
def _run_pytest_update_data(self, data_suite: str, *, max_attempts: int) -> str:
|
||
|
"""
|
||
|
Runs a suite of data test cases through 'pytest --update-data' until either tests pass
|
||
|
or until a maximum number of attempts (needed for incremental tests).
|
||
|
"""
|
||
|
p_test_data = Path(test_data_prefix)
|
||
|
p_root = p_test_data.parent.parent
|
||
|
p = p_test_data / f"check-meta-{uuid.uuid4()}.test"
|
||
|
assert not p.exists()
|
||
|
try:
|
||
|
p.write_text(textwrap.dedent(data_suite).lstrip())
|
||
|
|
||
|
test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}"
|
||
|
args = [sys.executable, "-m", "pytest", "-n", "0", "-s", "--update-data", test_nodeid]
|
||
|
cmd = shlex.join(args)
|
||
|
for i in range(max_attempts - 1, -1, -1):
|
||
|
res = subprocess.run(args, cwd=p_root)
|
||
|
if res.returncode == 0:
|
||
|
break
|
||
|
print(f"`{cmd}` returned {res.returncode}: {i} attempts remaining")
|
||
|
|
||
|
return p.read_text()
|
||
|
finally:
|
||
|
p.unlink()
|
||
|
|
||
|
def test_update_data(self) -> None:
|
||
|
# Note: We test multiple testcases rather than 'test case per test case'
|
||
|
# so we could also exercise rewriting multiple testcases at once.
|
||
|
actual = self._run_pytest_update_data(
|
||
|
"""
|
||
|
[case testCorrect]
|
||
|
s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testWrong]
|
||
|
s: str = 42 # E: wrong error
|
||
|
|
||
|
[case testXfail-xfail]
|
||
|
s: str = 42 # E: wrong error
|
||
|
|
||
|
[case testWrongMultiline]
|
||
|
s: str = 42 # E: foo \
|
||
|
# N: bar
|
||
|
|
||
|
[case testMissingMultiline]
|
||
|
s: str = 42; i: int = 'foo'
|
||
|
|
||
|
[case testExtraneous]
|
||
|
s: str = 'foo' # E: wrong error
|
||
|
|
||
|
[case testExtraneousMultiline]
|
||
|
s: str = 'foo' # E: foo \
|
||
|
# E: bar
|
||
|
|
||
|
[case testExtraneousMultilineNonError]
|
||
|
s: str = 'foo' # W: foo \
|
||
|
# N: bar
|
||
|
|
||
|
[case testOutCorrect]
|
||
|
s: str = 42
|
||
|
[out]
|
||
|
main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testOutWrong]
|
||
|
s: str = 42
|
||
|
[out]
|
||
|
main:1: error: foobar
|
||
|
|
||
|
[case testOutWrongIncremental]
|
||
|
s: str = 42
|
||
|
[out]
|
||
|
main:1: error: foobar
|
||
|
[out2]
|
||
|
main:1: error: foobar
|
||
|
|
||
|
[case testWrongMultipleFiles]
|
||
|
import a, b
|
||
|
s: str = 42 # E: foo
|
||
|
[file a.py]
|
||
|
s1: str = 42 # E: bar
|
||
|
[file b.py]
|
||
|
s2: str = 43 # E: baz
|
||
|
[builtins fixtures/list.pyi]
|
||
|
""",
|
||
|
max_attempts=3,
|
||
|
)
|
||
|
|
||
|
# Assert
|
||
|
expected = """
|
||
|
[case testCorrect]
|
||
|
s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testWrong]
|
||
|
s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testXfail-xfail]
|
||
|
s: str = 42 # E: wrong error
|
||
|
|
||
|
[case testWrongMultiline]
|
||
|
s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testMissingMultiline]
|
||
|
s: str = 42; i: int = 'foo' # E: Incompatible types in assignment (expression has type "int", variable has type "str") \\
|
||
|
# E: Incompatible types in assignment (expression has type "str", variable has type "int")
|
||
|
|
||
|
[case testExtraneous]
|
||
|
s: str = 'foo'
|
||
|
|
||
|
[case testExtraneousMultiline]
|
||
|
s: str = 'foo'
|
||
|
|
||
|
[case testExtraneousMultilineNonError]
|
||
|
s: str = 'foo'
|
||
|
|
||
|
[case testOutCorrect]
|
||
|
s: str = 42
|
||
|
[out]
|
||
|
main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testOutWrong]
|
||
|
s: str = 42
|
||
|
[out]
|
||
|
main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testOutWrongIncremental]
|
||
|
s: str = 42
|
||
|
[out]
|
||
|
main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
[out2]
|
||
|
main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
|
||
|
[case testWrongMultipleFiles]
|
||
|
import a, b
|
||
|
s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
[file a.py]
|
||
|
s1: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
[file b.py]
|
||
|
s2: str = 43 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
|
||
|
[builtins fixtures/list.pyi]
|
||
|
"""
|
||
|
assert actual == textwrap.dedent(expected).lstrip()
|