377 lines
13 KiB
Python
377 lines
13 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import os
|
||
|
|
import shutil
|
||
|
|
import tempfile
|
||
|
|
import unittest
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list
|
||
|
|
from mypy.fscache import FileSystemCache
|
||
|
|
from mypy.modulefinder import BuildSource
|
||
|
|
from mypy.options import Options
|
||
|
|
|
||
|
|
|
||
|
|
class FakeFSCache(FileSystemCache):
|
||
|
|
def __init__(self, files: set[str]) -> None:
|
||
|
|
self.files = {os.path.abspath(f) for f in files}
|
||
|
|
|
||
|
|
def isfile(self, file: str) -> bool:
|
||
|
|
return file in self.files
|
||
|
|
|
||
|
|
def isdir(self, dir: str) -> bool:
|
||
|
|
if not dir.endswith(os.sep):
|
||
|
|
dir += os.sep
|
||
|
|
return any(f.startswith(dir) for f in self.files)
|
||
|
|
|
||
|
|
def listdir(self, dir: str) -> list[str]:
|
||
|
|
if not dir.endswith(os.sep):
|
||
|
|
dir += os.sep
|
||
|
|
return list({f[len(dir) :].split(os.sep)[0] for f in self.files if f.startswith(dir)})
|
||
|
|
|
||
|
|
def init_under_package_root(self, file: str) -> bool:
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def normalise_path(path: str) -> str:
|
||
|
|
path = os.path.splitdrive(path)[1]
|
||
|
|
path = path.replace(os.sep, "/")
|
||
|
|
return path
|
||
|
|
|
||
|
|
|
||
|
|
def normalise_build_source_list(sources: list[BuildSource]) -> list[tuple[str, str | None]]:
|
||
|
|
return sorted(
|
||
|
|
(s.module, (normalise_path(s.base_dir) if s.base_dir is not None else None))
|
||
|
|
for s in sources
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def crawl(finder: SourceFinder, f: str) -> tuple[str, str]:
|
||
|
|
module, base_dir = finder.crawl_up(f)
|
||
|
|
return module, normalise_path(base_dir)
|
||
|
|
|
||
|
|
|
||
|
|
def find_sources_in_dir(finder: SourceFinder, f: str) -> list[tuple[str, str | None]]:
|
||
|
|
return normalise_build_source_list(finder.find_sources_in_dir(os.path.abspath(f)))
|
||
|
|
|
||
|
|
|
||
|
|
def find_sources(
|
||
|
|
paths: list[str], options: Options, fscache: FileSystemCache
|
||
|
|
) -> list[tuple[str, str | None]]:
|
||
|
|
paths = [os.path.abspath(p) for p in paths]
|
||
|
|
return normalise_build_source_list(create_source_list(paths, options, fscache))
|
||
|
|
|
||
|
|
|
||
|
|
class SourceFinderSuite(unittest.TestCase):
|
||
|
|
def setUp(self) -> None:
|
||
|
|
self.tempdir = tempfile.mkdtemp()
|
||
|
|
self.oldcwd = os.getcwd()
|
||
|
|
os.chdir(self.tempdir)
|
||
|
|
|
||
|
|
def tearDown(self) -> None:
|
||
|
|
os.chdir(self.oldcwd)
|
||
|
|
shutil.rmtree(self.tempdir)
|
||
|
|
|
||
|
|
def test_crawl_no_namespace(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = False
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/setup.py") == ("setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/setup.py") == ("setup", "/a")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/setup.py") == ("a.setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
|
||
|
|
|
||
|
|
finder = SourceFinder(
|
||
|
|
FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options
|
||
|
|
)
|
||
|
|
assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b")
|
||
|
|
|
||
|
|
def test_crawl_namespace(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = True
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/setup.py") == ("setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/setup.py") == ("setup", "/a")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/setup.py") == ("a.setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/setup.py") == ("a.b.setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(
|
||
|
|
FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options
|
||
|
|
)
|
||
|
|
assert crawl(finder, "/a/b/c/setup.py") == ("a.b.c.setup", "/")
|
||
|
|
|
||
|
|
def test_crawl_namespace_explicit_base(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = True
|
||
|
|
options.explicit_package_bases = True
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/setup.py") == ("setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/setup.py") == ("setup", "/a")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/setup.py") == ("a.setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/setup.py") == ("a.b.setup", "/")
|
||
|
|
|
||
|
|
finder = SourceFinder(
|
||
|
|
FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options
|
||
|
|
)
|
||
|
|
assert crawl(finder, "/a/b/c/setup.py") == ("a.b.c.setup", "/")
|
||
|
|
|
||
|
|
# set mypy path, so we actually have some explicit base dirs
|
||
|
|
options.mypy_path = ["/a/b"]
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b")
|
||
|
|
|
||
|
|
finder = SourceFinder(
|
||
|
|
FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options
|
||
|
|
)
|
||
|
|
assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b")
|
||
|
|
|
||
|
|
options.mypy_path = ["/a/b", "/a/b/c"]
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options)
|
||
|
|
assert crawl(finder, "/a/b/c/setup.py") == ("setup", "/a/b/c")
|
||
|
|
|
||
|
|
def test_crawl_namespace_multi_dir(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = True
|
||
|
|
options.explicit_package_bases = True
|
||
|
|
options.mypy_path = ["/a", "/b"]
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options)
|
||
|
|
assert crawl(finder, "/a/pkg/a.py") == ("pkg.a", "/a")
|
||
|
|
assert crawl(finder, "/b/pkg/b.py") == ("pkg.b", "/b")
|
||
|
|
|
||
|
|
def test_find_sources_in_dir_no_namespace(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = False
|
||
|
|
|
||
|
|
files = {
|
||
|
|
"/pkg/a1/b/c/d/e.py",
|
||
|
|
"/pkg/a1/b/f.py",
|
||
|
|
"/pkg/a2/__init__.py",
|
||
|
|
"/pkg/a2/b/c/d/e.py",
|
||
|
|
"/pkg/a2/b/f.py",
|
||
|
|
}
|
||
|
|
finder = SourceFinder(FakeFSCache(files), options)
|
||
|
|
assert find_sources_in_dir(finder, "/") == [
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("e", "/pkg/a1/b/c/d"),
|
||
|
|
("e", "/pkg/a2/b/c/d"),
|
||
|
|
("f", "/pkg/a1/b"),
|
||
|
|
("f", "/pkg/a2/b"),
|
||
|
|
]
|
||
|
|
|
||
|
|
def test_find_sources_in_dir_namespace(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = True
|
||
|
|
|
||
|
|
files = {
|
||
|
|
"/pkg/a1/b/c/d/e.py",
|
||
|
|
"/pkg/a1/b/f.py",
|
||
|
|
"/pkg/a2/__init__.py",
|
||
|
|
"/pkg/a2/b/c/d/e.py",
|
||
|
|
"/pkg/a2/b/f.py",
|
||
|
|
}
|
||
|
|
finder = SourceFinder(FakeFSCache(files), options)
|
||
|
|
assert find_sources_in_dir(finder, "/") == [
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("a2.b.c.d.e", "/pkg"),
|
||
|
|
("a2.b.f", "/pkg"),
|
||
|
|
("e", "/pkg/a1/b/c/d"),
|
||
|
|
("f", "/pkg/a1/b"),
|
||
|
|
]
|
||
|
|
|
||
|
|
def test_find_sources_in_dir_namespace_explicit_base(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = True
|
||
|
|
options.explicit_package_bases = True
|
||
|
|
options.mypy_path = ["/"]
|
||
|
|
|
||
|
|
files = {
|
||
|
|
"/pkg/a1/b/c/d/e.py",
|
||
|
|
"/pkg/a1/b/f.py",
|
||
|
|
"/pkg/a2/__init__.py",
|
||
|
|
"/pkg/a2/b/c/d/e.py",
|
||
|
|
"/pkg/a2/b/f.py",
|
||
|
|
}
|
||
|
|
finder = SourceFinder(FakeFSCache(files), options)
|
||
|
|
assert find_sources_in_dir(finder, "/") == [
|
||
|
|
("pkg.a1.b.c.d.e", "/"),
|
||
|
|
("pkg.a1.b.f", "/"),
|
||
|
|
("pkg.a2", "/"),
|
||
|
|
("pkg.a2.b.c.d.e", "/"),
|
||
|
|
("pkg.a2.b.f", "/"),
|
||
|
|
]
|
||
|
|
|
||
|
|
options.mypy_path = ["/pkg"]
|
||
|
|
finder = SourceFinder(FakeFSCache(files), options)
|
||
|
|
assert find_sources_in_dir(finder, "/") == [
|
||
|
|
("a1.b.c.d.e", "/pkg"),
|
||
|
|
("a1.b.f", "/pkg"),
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("a2.b.c.d.e", "/pkg"),
|
||
|
|
("a2.b.f", "/pkg"),
|
||
|
|
]
|
||
|
|
|
||
|
|
def test_find_sources_in_dir_namespace_multi_dir(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = True
|
||
|
|
options.explicit_package_bases = True
|
||
|
|
options.mypy_path = ["/a", "/b"]
|
||
|
|
|
||
|
|
finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options)
|
||
|
|
assert find_sources_in_dir(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")]
|
||
|
|
|
||
|
|
def test_find_sources_exclude(self) -> None:
|
||
|
|
options = Options()
|
||
|
|
options.namespace_packages = True
|
||
|
|
|
||
|
|
# default
|
||
|
|
for excluded_dir in ["site-packages", ".whatever", "node_modules", ".x/.z"]:
|
||
|
|
fscache = FakeFSCache({"/dir/a.py", f"/dir/venv/{excluded_dir}/b.py"})
|
||
|
|
assert find_sources(["/"], options, fscache) == [("a", "/dir")]
|
||
|
|
with pytest.raises(InvalidSourceList):
|
||
|
|
find_sources(["/dir/venv/"], options, fscache)
|
||
|
|
assert find_sources([f"/dir/venv/{excluded_dir}"], options, fscache) == [
|
||
|
|
("b", f"/dir/venv/{excluded_dir}")
|
||
|
|
]
|
||
|
|
assert find_sources([f"/dir/venv/{excluded_dir}/b.py"], options, fscache) == [
|
||
|
|
("b", f"/dir/venv/{excluded_dir}")
|
||
|
|
]
|
||
|
|
|
||
|
|
files = {
|
||
|
|
"/pkg/a1/b/c/d/e.py",
|
||
|
|
"/pkg/a1/b/f.py",
|
||
|
|
"/pkg/a2/__init__.py",
|
||
|
|
"/pkg/a2/b/c/d/e.py",
|
||
|
|
"/pkg/a2/b/f.py",
|
||
|
|
}
|
||
|
|
|
||
|
|
# file name
|
||
|
|
options.exclude = [r"/f\.py$"]
|
||
|
|
fscache = FakeFSCache(files)
|
||
|
|
assert find_sources(["/"], options, fscache) == [
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("a2.b.c.d.e", "/pkg"),
|
||
|
|
("e", "/pkg/a1/b/c/d"),
|
||
|
|
]
|
||
|
|
assert find_sources(["/pkg/a1/b/f.py"], options, fscache) == [("f", "/pkg/a1/b")]
|
||
|
|
assert find_sources(["/pkg/a2/b/f.py"], options, fscache) == [("a2.b.f", "/pkg")]
|
||
|
|
|
||
|
|
# directory name
|
||
|
|
options.exclude = ["/a1/"]
|
||
|
|
fscache = FakeFSCache(files)
|
||
|
|
assert find_sources(["/"], options, fscache) == [
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("a2.b.c.d.e", "/pkg"),
|
||
|
|
("a2.b.f", "/pkg"),
|
||
|
|
]
|
||
|
|
with pytest.raises(InvalidSourceList):
|
||
|
|
find_sources(["/pkg/a1"], options, fscache)
|
||
|
|
with pytest.raises(InvalidSourceList):
|
||
|
|
find_sources(["/pkg/a1/"], options, fscache)
|
||
|
|
with pytest.raises(InvalidSourceList):
|
||
|
|
find_sources(["/pkg/a1/b"], options, fscache)
|
||
|
|
|
||
|
|
options.exclude = ["/a1/$"]
|
||
|
|
assert find_sources(["/pkg/a1"], options, fscache) == [
|
||
|
|
("e", "/pkg/a1/b/c/d"),
|
||
|
|
("f", "/pkg/a1/b"),
|
||
|
|
]
|
||
|
|
|
||
|
|
# paths
|
||
|
|
options.exclude = ["/pkg/a1/"]
|
||
|
|
fscache = FakeFSCache(files)
|
||
|
|
assert find_sources(["/"], options, fscache) == [
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("a2.b.c.d.e", "/pkg"),
|
||
|
|
("a2.b.f", "/pkg"),
|
||
|
|
]
|
||
|
|
with pytest.raises(InvalidSourceList):
|
||
|
|
find_sources(["/pkg/a1"], options, fscache)
|
||
|
|
|
||
|
|
# OR two patterns together
|
||
|
|
for orred in [["/(a1|a3)/"], ["a1", "a3"], ["a3", "a1"]]:
|
||
|
|
options.exclude = orred
|
||
|
|
fscache = FakeFSCache(files)
|
||
|
|
assert find_sources(["/"], options, fscache) == [
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("a2.b.c.d.e", "/pkg"),
|
||
|
|
("a2.b.f", "/pkg"),
|
||
|
|
]
|
||
|
|
|
||
|
|
options.exclude = ["b/c/"]
|
||
|
|
fscache = FakeFSCache(files)
|
||
|
|
assert find_sources(["/"], options, fscache) == [
|
||
|
|
("a2", "/pkg"),
|
||
|
|
("a2.b.f", "/pkg"),
|
||
|
|
("f", "/pkg/a1/b"),
|
||
|
|
]
|
||
|
|
|
||
|
|
# nothing should be ignored as a result of this
|
||
|
|
big_exclude1 = [
|
||
|
|
"/pkg/a/",
|
||
|
|
"/2",
|
||
|
|
"/1",
|
||
|
|
"/pk/",
|
||
|
|
"/kg",
|
||
|
|
"/g.py",
|
||
|
|
"/bc",
|
||
|
|
"/xxx/pkg/a2/b/f.py",
|
||
|
|
"xxx/pkg/a2/b/f.py",
|
||
|
|
]
|
||
|
|
big_exclude2 = ["|".join(big_exclude1)]
|
||
|
|
for big_exclude in [big_exclude1, big_exclude2]:
|
||
|
|
options.exclude = big_exclude
|
||
|
|
fscache = FakeFSCache(files)
|
||
|
|
assert len(find_sources(["/"], options, fscache)) == len(files)
|
||
|
|
|
||
|
|
files = {
|
||
|
|
"pkg/a1/b/c/d/e.py",
|
||
|
|
"pkg/a1/b/f.py",
|
||
|
|
"pkg/a2/__init__.py",
|
||
|
|
"pkg/a2/b/c/d/e.py",
|
||
|
|
"pkg/a2/b/f.py",
|
||
|
|
}
|
||
|
|
fscache = FakeFSCache(files)
|
||
|
|
assert len(find_sources(["."], options, fscache)) == len(files)
|