245 lines
9.3 KiB
Python
245 lines
9.3 KiB
Python
|
"""Routines for finding the sources that mypy will check"""
|
||
|
|
|
||
|
|
import functools
|
||
|
|
import os
|
||
|
|
|
||
|
|
from typing import List, Sequence, Set, Tuple, Optional
|
||
|
|
from typing_extensions import Final
|
||
|
|
|
||
|
|
from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_exclude
|
||
|
|
from mypy.fscache import FileSystemCache
|
||
|
|
from mypy.options import Options
|
||
|
|
|
||
|
|
PY_EXTENSIONS: Final = tuple(PYTHON_EXTENSIONS)
|
||
|
|
|
||
|
|
|
||
|
|
class InvalidSourceList(Exception):
|
||
|
|
"""Exception indicating a problem in the list of sources given to mypy."""
|
||
|
|
|
||
|
|
|
||
|
|
def create_source_list(paths: Sequence[str], options: Options,
|
||
|
|
fscache: Optional[FileSystemCache] = None,
|
||
|
|
allow_empty_dir: bool = False) -> List[BuildSource]:
|
||
|
|
"""From a list of source files/directories, makes a list of BuildSources.
|
||
|
|
|
||
|
|
Raises InvalidSourceList on errors.
|
||
|
|
"""
|
||
|
|
fscache = fscache or FileSystemCache()
|
||
|
|
finder = SourceFinder(fscache, options)
|
||
|
|
|
||
|
|
sources = []
|
||
|
|
for path in paths:
|
||
|
|
path = os.path.normpath(path)
|
||
|
|
if path.endswith(PY_EXTENSIONS):
|
||
|
|
# Can raise InvalidSourceList if a directory doesn't have a valid module name.
|
||
|
|
name, base_dir = finder.crawl_up(path)
|
||
|
|
sources.append(BuildSource(path, name, None, base_dir))
|
||
|
|
elif fscache.isdir(path):
|
||
|
|
sub_sources = finder.find_sources_in_dir(path)
|
||
|
|
if not sub_sources and not allow_empty_dir:
|
||
|
|
raise InvalidSourceList(
|
||
|
|
f"There are no .py[i] files in directory '{path}'"
|
||
|
|
)
|
||
|
|
sources.extend(sub_sources)
|
||
|
|
else:
|
||
|
|
mod = os.path.basename(path) if options.scripts_are_modules else None
|
||
|
|
sources.append(BuildSource(path, mod, None))
|
||
|
|
return sources
|
||
|
|
|
||
|
|
|
||
|
|
def keyfunc(name: str) -> Tuple[bool, int, str]:
|
||
|
|
"""Determines sort order for directory listing.
|
||
|
|
|
||
|
|
The desirable properties are:
|
||
|
|
1) foo < foo.pyi < foo.py
|
||
|
|
2) __init__.py[i] < foo
|
||
|
|
"""
|
||
|
|
base, suffix = os.path.splitext(name)
|
||
|
|
for i, ext in enumerate(PY_EXTENSIONS):
|
||
|
|
if suffix == ext:
|
||
|
|
return (base != "__init__", i, base)
|
||
|
|
return (base != "__init__", -1, name)
|
||
|
|
|
||
|
|
|
||
|
|
def normalise_package_base(root: str) -> str:
|
||
|
|
if not root:
|
||
|
|
root = os.curdir
|
||
|
|
root = os.path.abspath(root)
|
||
|
|
if root.endswith(os.sep):
|
||
|
|
root = root[:-1]
|
||
|
|
return root
|
||
|
|
|
||
|
|
|
||
|
|
def get_explicit_package_bases(options: Options) -> Optional[List[str]]:
|
||
|
|
"""Returns explicit package bases to use if the option is enabled, or None if disabled.
|
||
|
|
|
||
|
|
We currently use MYPYPATH and the current directory as the package bases. In the future,
|
||
|
|
when --namespace-packages is the default could also use the values passed with the
|
||
|
|
--package-root flag, see #9632.
|
||
|
|
|
||
|
|
Values returned are normalised so we can use simple string comparisons in
|
||
|
|
SourceFinder.is_explicit_package_base
|
||
|
|
"""
|
||
|
|
if not options.explicit_package_bases:
|
||
|
|
return None
|
||
|
|
roots = mypy_path() + options.mypy_path + [os.getcwd()]
|
||
|
|
return [normalise_package_base(root) for root in roots]
|
||
|
|
|
||
|
|
|
||
|
|
class SourceFinder:
|
||
|
|
def __init__(self, fscache: FileSystemCache, options: Options) -> None:
|
||
|
|
self.fscache = fscache
|
||
|
|
self.explicit_package_bases = get_explicit_package_bases(options)
|
||
|
|
self.namespace_packages = options.namespace_packages
|
||
|
|
self.exclude = options.exclude
|
||
|
|
self.verbosity = options.verbosity
|
||
|
|
|
||
|
|
def is_explicit_package_base(self, path: str) -> bool:
|
||
|
|
assert self.explicit_package_bases
|
||
|
|
return normalise_package_base(path) in self.explicit_package_bases
|
||
|
|
|
||
|
|
def find_sources_in_dir(self, path: str) -> List[BuildSource]:
|
||
|
|
sources = []
|
||
|
|
|
||
|
|
seen: Set[str] = set()
|
||
|
|
names = sorted(self.fscache.listdir(path), key=keyfunc)
|
||
|
|
for name in names:
|
||
|
|
# Skip certain names altogether
|
||
|
|
if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."):
|
||
|
|
continue
|
||
|
|
subpath = os.path.join(path, name)
|
||
|
|
|
||
|
|
if matches_exclude(
|
||
|
|
subpath, self.exclude, self.fscache, self.verbosity >= 2
|
||
|
|
):
|
||
|
|
continue
|
||
|
|
|
||
|
|
if self.fscache.isdir(subpath):
|
||
|
|
sub_sources = self.find_sources_in_dir(subpath)
|
||
|
|
if sub_sources:
|
||
|
|
seen.add(name)
|
||
|
|
sources.extend(sub_sources)
|
||
|
|
else:
|
||
|
|
stem, suffix = os.path.splitext(name)
|
||
|
|
if stem not in seen and suffix in PY_EXTENSIONS:
|
||
|
|
seen.add(stem)
|
||
|
|
module, base_dir = self.crawl_up(subpath)
|
||
|
|
sources.append(BuildSource(subpath, module, None, base_dir))
|
||
|
|
|
||
|
|
return sources
|
||
|
|
|
||
|
|
def crawl_up(self, path: str) -> Tuple[str, str]:
|
||
|
|
"""Given a .py[i] filename, return module and base directory.
|
||
|
|
|
||
|
|
For example, given "xxx/yyy/foo/bar.py", we might return something like:
|
||
|
|
("foo.bar", "xxx/yyy")
|
||
|
|
|
||
|
|
If namespace packages is off, we crawl upwards until we find a directory without
|
||
|
|
an __init__.py
|
||
|
|
|
||
|
|
If namespace packages is on, we crawl upwards until the nearest explicit base directory.
|
||
|
|
Failing that, we return one past the highest directory containing an __init__.py
|
||
|
|
|
||
|
|
We won't crawl past directories with invalid package names.
|
||
|
|
The base directory returned is an absolute path.
|
||
|
|
"""
|
||
|
|
path = os.path.abspath(path)
|
||
|
|
parent, filename = os.path.split(path)
|
||
|
|
|
||
|
|
module_name = strip_py(filename) or filename
|
||
|
|
|
||
|
|
parent_module, base_dir = self.crawl_up_dir(parent)
|
||
|
|
if module_name == "__init__":
|
||
|
|
return parent_module, base_dir
|
||
|
|
|
||
|
|
# Note that module_name might not actually be a valid identifier, but that's okay
|
||
|
|
# Ignoring this possibility sidesteps some search path confusion
|
||
|
|
module = module_join(parent_module, module_name)
|
||
|
|
return module, base_dir
|
||
|
|
|
||
|
|
def crawl_up_dir(self, dir: str) -> Tuple[str, str]:
|
||
|
|
return self._crawl_up_helper(dir) or ("", dir)
|
||
|
|
|
||
|
|
@functools.lru_cache() # noqa: B019
|
||
|
|
def _crawl_up_helper(self, dir: str) -> Optional[Tuple[str, str]]:
|
||
|
|
"""Given a directory, maybe returns module and base directory.
|
||
|
|
|
||
|
|
We return a non-None value if we were able to find something clearly intended as a base
|
||
|
|
directory (as adjudicated by being an explicit base directory or by containing a package
|
||
|
|
with __init__.py).
|
||
|
|
|
||
|
|
This distinction is necessary for namespace packages, so that we know when to treat
|
||
|
|
ourselves as a subpackage.
|
||
|
|
"""
|
||
|
|
# stop crawling if we're an explicit base directory
|
||
|
|
if self.explicit_package_bases is not None and self.is_explicit_package_base(dir):
|
||
|
|
return "", dir
|
||
|
|
|
||
|
|
parent, name = os.path.split(dir)
|
||
|
|
if name.endswith('-stubs'):
|
||
|
|
name = name[:-6] # PEP-561 stub-only directory
|
||
|
|
|
||
|
|
# recurse if there's an __init__.py
|
||
|
|
init_file = self.get_init_file(dir)
|
||
|
|
if init_file is not None:
|
||
|
|
if not name.isidentifier():
|
||
|
|
# in most cases the directory name is invalid, we'll just stop crawling upwards
|
||
|
|
# but if there's an __init__.py in the directory, something is messed up
|
||
|
|
raise InvalidSourceList(f"{name} is not a valid Python package name")
|
||
|
|
# we're definitely a package, so we always return a non-None value
|
||
|
|
mod_prefix, base_dir = self.crawl_up_dir(parent)
|
||
|
|
return module_join(mod_prefix, name), base_dir
|
||
|
|
|
||
|
|
# stop crawling if we're out of path components or our name is an invalid identifier
|
||
|
|
if not name or not parent or not name.isidentifier():
|
||
|
|
return None
|
||
|
|
|
||
|
|
# stop crawling if namespace packages is off (since we don't have an __init__.py)
|
||
|
|
if not self.namespace_packages:
|
||
|
|
return None
|
||
|
|
|
||
|
|
# at this point: namespace packages is on, we don't have an __init__.py and we're not an
|
||
|
|
# explicit base directory
|
||
|
|
result = self._crawl_up_helper(parent)
|
||
|
|
if result is None:
|
||
|
|
# we're not an explicit base directory and we don't have an __init__.py
|
||
|
|
# and none of our parents are either, so return
|
||
|
|
return None
|
||
|
|
# one of our parents was an explicit base directory or had an __init__.py, so we're
|
||
|
|
# definitely a subpackage! chain our name to the module.
|
||
|
|
mod_prefix, base_dir = result
|
||
|
|
return module_join(mod_prefix, name), base_dir
|
||
|
|
|
||
|
|
def get_init_file(self, dir: str) -> Optional[str]:
|
||
|
|
"""Check whether a directory contains a file named __init__.py[i].
|
||
|
|
|
||
|
|
If so, return the file's name (with dir prefixed). If not, return None.
|
||
|
|
|
||
|
|
This prefers .pyi over .py (because of the ordering of PY_EXTENSIONS).
|
||
|
|
"""
|
||
|
|
for ext in PY_EXTENSIONS:
|
||
|
|
f = os.path.join(dir, '__init__' + ext)
|
||
|
|
if self.fscache.isfile(f):
|
||
|
|
return f
|
||
|
|
if ext == '.py' and self.fscache.init_under_package_root(f):
|
||
|
|
return f
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def module_join(parent: str, child: str) -> str:
|
||
|
|
"""Join module ids, accounting for a possibly empty parent."""
|
||
|
|
if parent:
|
||
|
|
return parent + '.' + child
|
||
|
|
return child
|
||
|
|
|
||
|
|
|
||
|
|
def strip_py(arg: str) -> Optional[str]:
|
||
|
|
"""Strip a trailing .py or .pyi suffix.
|
||
|
|
|
||
|
|
Return None if no such suffix is found.
|
||
|
|
"""
|
||
|
|
for ext in PY_EXTENSIONS:
|
||
|
|
if arg.endswith(ext):
|
||
|
|
return arg[:-len(ext)]
|
||
|
|
return None
|