568 lines
21 KiB
Python
568 lines
21 KiB
Python
import argparse
|
|
import configparser
|
|
import glob as fileglob
|
|
from io import StringIO
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
if sys.version_info >= (3, 11):
|
|
import tomllib
|
|
else:
|
|
import tomli as tomllib
|
|
|
|
from typing import (Any, Callable, Dict, List, Mapping, MutableMapping, Optional, Sequence,
|
|
TextIO, Tuple, Union)
|
|
from typing_extensions import Final, TypeAlias as _TypeAlias
|
|
|
|
from mypy import defaults
|
|
from mypy.options import Options, PER_MODULE_OPTIONS
|
|
|
|
_CONFIG_VALUE_TYPES: _TypeAlias = Union[
|
|
str, bool, int, float, Dict[str, str], List[str], Tuple[int, int],
|
|
]
|
|
_INI_PARSER_CALLABLE: _TypeAlias = Callable[[Any], _CONFIG_VALUE_TYPES]
|
|
|
|
|
|
def parse_version(v: Union[str, float]) -> Tuple[int, int]:
|
|
m = re.match(r'\A(\d)\.(\d+)\Z', str(v))
|
|
if not m:
|
|
raise argparse.ArgumentTypeError(
|
|
f"Invalid python version '{v}' (expected format: 'x.y')")
|
|
major, minor = int(m.group(1)), int(m.group(2))
|
|
if major == 2:
|
|
if minor != 7:
|
|
raise argparse.ArgumentTypeError(
|
|
f"Python 2.{minor} is not supported (must be 2.7)")
|
|
elif major == 3:
|
|
if minor < defaults.PYTHON3_VERSION_MIN[1]:
|
|
msg = "Python 3.{0} is not supported (must be {1}.{2} or higher)".format(
|
|
minor,
|
|
*defaults.PYTHON3_VERSION_MIN
|
|
)
|
|
|
|
if isinstance(v, float):
|
|
msg += ". You may need to put quotes around your Python version"
|
|
|
|
raise argparse.ArgumentTypeError(msg)
|
|
else:
|
|
raise argparse.ArgumentTypeError(
|
|
f"Python major version '{major}' out of range (must be 2 or 3)")
|
|
return major, minor
|
|
|
|
|
|
def try_split(v: Union[str, Sequence[str]], split_regex: str = '[,]') -> List[str]:
|
|
"""Split and trim a str or list of str into a list of str"""
|
|
if isinstance(v, str):
|
|
return [p.strip() for p in re.split(split_regex, v)]
|
|
|
|
return [p.strip() for p in v]
|
|
|
|
|
|
def expand_path(path: str) -> str:
|
|
"""Expand the user home directory and any environment variables contained within
|
|
the provided path.
|
|
"""
|
|
|
|
return os.path.expandvars(os.path.expanduser(path))
|
|
|
|
|
|
def str_or_array_as_list(v: Union[str, Sequence[str]]) -> List[str]:
|
|
if isinstance(v, str):
|
|
return [v.strip()] if v.strip() else []
|
|
return [p.strip() for p in v if p.strip()]
|
|
|
|
|
|
def split_and_match_files_list(paths: Sequence[str]) -> List[str]:
|
|
"""Take a list of files/directories (with support for globbing through the glob library).
|
|
|
|
Where a path/glob matches no file, we still include the raw path in the resulting list.
|
|
|
|
Returns a list of file paths
|
|
"""
|
|
expanded_paths = []
|
|
|
|
for path in paths:
|
|
path = expand_path(path.strip())
|
|
globbed_files = fileglob.glob(path, recursive=True)
|
|
if globbed_files:
|
|
expanded_paths.extend(globbed_files)
|
|
else:
|
|
expanded_paths.append(path)
|
|
|
|
return expanded_paths
|
|
|
|
|
|
def split_and_match_files(paths: str) -> List[str]:
|
|
"""Take a string representing a list of files/directories (with support for globbing
|
|
through the glob library).
|
|
|
|
Where a path/glob matches no file, we still include the raw path in the resulting list.
|
|
|
|
Returns a list of file paths
|
|
"""
|
|
|
|
return split_and_match_files_list(paths.split(','))
|
|
|
|
|
|
def check_follow_imports(choice: str) -> str:
|
|
choices = ['normal', 'silent', 'skip', 'error']
|
|
if choice not in choices:
|
|
raise argparse.ArgumentTypeError(
|
|
"invalid choice '{}' (choose from {})".format(
|
|
choice,
|
|
', '.join(f"'{x}'" for x in choices)))
|
|
return choice
|
|
|
|
|
|
# For most options, the type of the default value set in options.py is
|
|
# sufficient, and we don't have to do anything here. This table
|
|
# exists to specify types for values initialized to None or container
|
|
# types.
|
|
ini_config_types: Final[Dict[str, _INI_PARSER_CALLABLE]] = {
|
|
'python_version': parse_version,
|
|
'strict_optional_whitelist': lambda s: s.split(),
|
|
'custom_typing_module': str,
|
|
'custom_typeshed_dir': expand_path,
|
|
'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)],
|
|
'files': split_and_match_files,
|
|
'quickstart_file': expand_path,
|
|
'junit_xml': expand_path,
|
|
# These two are for backwards compatibility
|
|
'silent_imports': bool,
|
|
'almost_silent': bool,
|
|
'follow_imports': check_follow_imports,
|
|
'no_site_packages': bool,
|
|
'plugins': lambda s: [p.strip() for p in s.split(',')],
|
|
'always_true': lambda s: [p.strip() for p in s.split(',')],
|
|
'always_false': lambda s: [p.strip() for p in s.split(',')],
|
|
'disable_error_code': lambda s: [p.strip() for p in s.split(',')],
|
|
'enable_error_code': lambda s: [p.strip() for p in s.split(',')],
|
|
'package_root': lambda s: [p.strip() for p in s.split(',')],
|
|
'cache_dir': expand_path,
|
|
'python_executable': expand_path,
|
|
'strict': bool,
|
|
'exclude': lambda s: [s.strip()],
|
|
}
|
|
|
|
# Reuse the ini_config_types and overwrite the diff
|
|
toml_config_types: Final[Dict[str, _INI_PARSER_CALLABLE]] = ini_config_types.copy()
|
|
toml_config_types.update({
|
|
'python_version': parse_version,
|
|
'strict_optional_whitelist': try_split,
|
|
'mypy_path': lambda s: [expand_path(p) for p in try_split(s, '[,:]')],
|
|
'files': lambda s: split_and_match_files_list(try_split(s)),
|
|
'follow_imports': lambda s: check_follow_imports(str(s)),
|
|
'plugins': try_split,
|
|
'always_true': try_split,
|
|
'always_false': try_split,
|
|
'disable_error_code': try_split,
|
|
'enable_error_code': try_split,
|
|
'package_root': try_split,
|
|
'exclude': str_or_array_as_list,
|
|
})
|
|
|
|
|
|
def parse_config_file(options: Options, set_strict_flags: Callable[[], None],
|
|
filename: Optional[str],
|
|
stdout: Optional[TextIO] = None,
|
|
stderr: Optional[TextIO] = None) -> None:
|
|
"""Parse a config file into an Options object.
|
|
|
|
Errors are written to stderr but are not fatal.
|
|
|
|
If filename is None, fall back to default config files.
|
|
"""
|
|
stdout = stdout or sys.stdout
|
|
stderr = stderr or sys.stderr
|
|
|
|
if filename is not None:
|
|
config_files: Tuple[str, ...] = (filename,)
|
|
else:
|
|
config_files = tuple(map(os.path.expanduser, defaults.CONFIG_FILES))
|
|
|
|
config_parser = configparser.RawConfigParser()
|
|
|
|
for config_file in config_files:
|
|
if not os.path.exists(config_file):
|
|
continue
|
|
try:
|
|
if is_toml(config_file):
|
|
with open(config_file, "rb") as f:
|
|
toml_data = tomllib.load(f)
|
|
# Filter down to just mypy relevant toml keys
|
|
toml_data = toml_data.get('tool', {})
|
|
if 'mypy' not in toml_data:
|
|
continue
|
|
toml_data = {'mypy': toml_data['mypy']}
|
|
parser: MutableMapping[str, Any] = destructure_overrides(toml_data)
|
|
config_types = toml_config_types
|
|
else:
|
|
config_parser.read(config_file)
|
|
parser = config_parser
|
|
config_types = ini_config_types
|
|
except (tomllib.TOMLDecodeError, configparser.Error, ConfigTOMLValueError) as err:
|
|
print(f"{config_file}: {err}", file=stderr)
|
|
else:
|
|
if config_file in defaults.SHARED_CONFIG_FILES and 'mypy' not in parser:
|
|
continue
|
|
file_read = config_file
|
|
options.config_file = file_read
|
|
break
|
|
else:
|
|
return
|
|
|
|
os.environ['MYPY_CONFIG_FILE_DIR'] = os.path.dirname(
|
|
os.path.abspath(config_file))
|
|
|
|
if 'mypy' not in parser:
|
|
if filename or file_read not in defaults.SHARED_CONFIG_FILES:
|
|
print(f"{file_read}: No [mypy] section in config file", file=stderr)
|
|
else:
|
|
section = parser['mypy']
|
|
prefix = f"{file_read}: [mypy]: "
|
|
updates, report_dirs = parse_section(
|
|
prefix, options, set_strict_flags, section, config_types, stderr)
|
|
for k, v in updates.items():
|
|
setattr(options, k, v)
|
|
options.report_dirs.update(report_dirs)
|
|
|
|
for name, section in parser.items():
|
|
if name.startswith('mypy-'):
|
|
prefix = get_prefix(file_read, name)
|
|
updates, report_dirs = parse_section(
|
|
prefix, options, set_strict_flags, section, config_types, stderr)
|
|
if report_dirs:
|
|
print("%sPer-module sections should not specify reports (%s)" %
|
|
(prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
|
|
file=stderr)
|
|
if set(updates) - PER_MODULE_OPTIONS:
|
|
print("%sPer-module sections should only specify per-module flags (%s)" %
|
|
(prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))),
|
|
file=stderr)
|
|
updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS}
|
|
globs = name[5:]
|
|
for glob in globs.split(','):
|
|
# For backwards compatibility, replace (back)slashes with dots.
|
|
glob = glob.replace(os.sep, '.')
|
|
if os.altsep:
|
|
glob = glob.replace(os.altsep, '.')
|
|
|
|
if (any(c in glob for c in '?[]!') or
|
|
any('*' in x and x != '*' for x in glob.split('.'))):
|
|
print("%sPatterns must be fully-qualified module names, optionally "
|
|
"with '*' in some components (e.g spam.*.eggs.*)"
|
|
% prefix,
|
|
file=stderr)
|
|
else:
|
|
options.per_module_options[glob] = updates
|
|
|
|
|
|
def get_prefix(file_read: str, name: str) -> str:
|
|
if is_toml(file_read):
|
|
module_name_str = 'module = "%s"' % '-'.join(name.split('-')[1:])
|
|
else:
|
|
module_name_str = name
|
|
|
|
return f'{file_read}: [{module_name_str}]: '
|
|
|
|
|
|
def is_toml(filename: str) -> bool:
|
|
return filename.lower().endswith('.toml')
|
|
|
|
|
|
def destructure_overrides(toml_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Take the new [[tool.mypy.overrides]] section array in the pyproject.toml file,
|
|
and convert it back to a flatter structure that the existing config_parser can handle.
|
|
|
|
E.g. the following pyproject.toml file:
|
|
|
|
[[tool.mypy.overrides]]
|
|
module = [
|
|
"a.b",
|
|
"b.*"
|
|
]
|
|
disallow_untyped_defs = true
|
|
|
|
[[tool.mypy.overrides]]
|
|
module = 'c'
|
|
disallow_untyped_defs = false
|
|
|
|
Would map to the following config dict that it would have gotten from parsing an equivalent
|
|
ini file:
|
|
|
|
{
|
|
"mypy-a.b": {
|
|
disallow_untyped_defs = true,
|
|
},
|
|
"mypy-b.*": {
|
|
disallow_untyped_defs = true,
|
|
},
|
|
"mypy-c": {
|
|
disallow_untyped_defs: false,
|
|
},
|
|
}
|
|
"""
|
|
if 'overrides' not in toml_data['mypy']:
|
|
return toml_data
|
|
|
|
if not isinstance(toml_data['mypy']['overrides'], list):
|
|
raise ConfigTOMLValueError("tool.mypy.overrides sections must be an array. Please make "
|
|
"sure you are using double brackets like so: [[tool.mypy.overrides]]")
|
|
|
|
result = toml_data.copy()
|
|
for override in result['mypy']['overrides']:
|
|
if 'module' not in override:
|
|
raise ConfigTOMLValueError("toml config file contains a [[tool.mypy.overrides]] "
|
|
"section, but no module to override was specified.")
|
|
|
|
if isinstance(override['module'], str):
|
|
modules = [override['module']]
|
|
elif isinstance(override['module'], list):
|
|
modules = override['module']
|
|
else:
|
|
raise ConfigTOMLValueError("toml config file contains a [[tool.mypy.overrides]] "
|
|
"section with a module value that is not a string or a list of "
|
|
"strings")
|
|
|
|
for module in modules:
|
|
module_overrides = override.copy()
|
|
del module_overrides['module']
|
|
old_config_name = f'mypy-{module}'
|
|
if old_config_name not in result:
|
|
result[old_config_name] = module_overrides
|
|
else:
|
|
for new_key, new_value in module_overrides.items():
|
|
if (new_key in result[old_config_name] and
|
|
result[old_config_name][new_key] != new_value):
|
|
raise ConfigTOMLValueError("toml config file contains "
|
|
"[[tool.mypy.overrides]] sections with conflicting "
|
|
"values. Module '%s' has two different values for '%s'"
|
|
% (module, new_key))
|
|
result[old_config_name][new_key] = new_value
|
|
|
|
del result['mypy']['overrides']
|
|
return result
|
|
|
|
|
|
def parse_section(prefix: str, template: Options,
|
|
set_strict_flags: Callable[[], None],
|
|
section: Mapping[str, Any],
|
|
config_types: Dict[str, Any],
|
|
stderr: TextIO = sys.stderr
|
|
) -> Tuple[Dict[str, object], Dict[str, str]]:
|
|
"""Parse one section of a config file.
|
|
|
|
Returns a dict of option values encountered, and a dict of report directories.
|
|
"""
|
|
results: Dict[str, object] = {}
|
|
report_dirs: Dict[str, str] = {}
|
|
for key in section:
|
|
invert = False
|
|
options_key = key
|
|
if key in config_types:
|
|
ct = config_types[key]
|
|
else:
|
|
dv = None
|
|
# We have to keep new_semantic_analyzer in Options
|
|
# for plugin compatibility but it is not a valid option anymore.
|
|
assert hasattr(template, 'new_semantic_analyzer')
|
|
if key != 'new_semantic_analyzer':
|
|
dv = getattr(template, key, None)
|
|
if dv is None:
|
|
if key.endswith('_report'):
|
|
report_type = key[:-7].replace('_', '-')
|
|
if report_type in defaults.REPORTER_NAMES:
|
|
report_dirs[report_type] = str(section[key])
|
|
else:
|
|
print(f"{prefix}Unrecognized report type: {key}",
|
|
file=stderr)
|
|
continue
|
|
if key.startswith('x_'):
|
|
pass # Don't complain about `x_blah` flags
|
|
elif key.startswith('no_') and hasattr(template, key[3:]):
|
|
options_key = key[3:]
|
|
invert = True
|
|
elif key.startswith('allow') and hasattr(template, 'dis' + key):
|
|
options_key = 'dis' + key
|
|
invert = True
|
|
elif key.startswith('disallow') and hasattr(template, key[3:]):
|
|
options_key = key[3:]
|
|
invert = True
|
|
elif key == 'strict':
|
|
pass # Special handling below
|
|
else:
|
|
print(f"{prefix}Unrecognized option: {key} = {section[key]}",
|
|
file=stderr)
|
|
if invert:
|
|
dv = getattr(template, options_key, None)
|
|
else:
|
|
continue
|
|
ct = type(dv)
|
|
v: Any = None
|
|
try:
|
|
if ct is bool:
|
|
if isinstance(section, dict):
|
|
v = convert_to_boolean(section.get(key))
|
|
else:
|
|
v = section.getboolean(key) # type: ignore[attr-defined] # Until better stub
|
|
if invert:
|
|
v = not v
|
|
elif callable(ct):
|
|
if invert:
|
|
print(f"{prefix}Can not invert non-boolean key {options_key}",
|
|
file=stderr)
|
|
continue
|
|
try:
|
|
v = ct(section.get(key))
|
|
except argparse.ArgumentTypeError as err:
|
|
print(f"{prefix}{key}: {err}", file=stderr)
|
|
continue
|
|
else:
|
|
print(f"{prefix}Don't know what type {key} should have", file=stderr)
|
|
continue
|
|
except ValueError as err:
|
|
print(f"{prefix}{key}: {err}", file=stderr)
|
|
continue
|
|
if key == 'strict':
|
|
if v:
|
|
set_strict_flags()
|
|
continue
|
|
if key == 'silent_imports':
|
|
print("%ssilent_imports has been replaced by "
|
|
"ignore_missing_imports=True; follow_imports=skip" % prefix, file=stderr)
|
|
if v:
|
|
if 'ignore_missing_imports' not in results:
|
|
results['ignore_missing_imports'] = True
|
|
if 'follow_imports' not in results:
|
|
results['follow_imports'] = 'skip'
|
|
if key == 'almost_silent':
|
|
print("%salmost_silent has been replaced by "
|
|
"follow_imports=error" % prefix, file=stderr)
|
|
if v:
|
|
if 'follow_imports' not in results:
|
|
results['follow_imports'] = 'error'
|
|
results[options_key] = v
|
|
return results, report_dirs
|
|
|
|
|
|
def convert_to_boolean(value: Optional[Any]) -> bool:
|
|
"""Return a boolean value translating from other types if necessary."""
|
|
if isinstance(value, bool):
|
|
return value
|
|
if not isinstance(value, str):
|
|
value = str(value)
|
|
if value.lower() not in configparser.RawConfigParser.BOOLEAN_STATES:
|
|
raise ValueError(f'Not a boolean: {value}')
|
|
return configparser.RawConfigParser.BOOLEAN_STATES[value.lower()]
|
|
|
|
|
|
def split_directive(s: str) -> Tuple[List[str], List[str]]:
|
|
"""Split s on commas, except during quoted sections.
|
|
|
|
Returns the parts and a list of error messages."""
|
|
parts = []
|
|
cur: List[str] = []
|
|
errors = []
|
|
i = 0
|
|
while i < len(s):
|
|
if s[i] == ',':
|
|
parts.append(''.join(cur).strip())
|
|
cur = []
|
|
elif s[i] == '"':
|
|
i += 1
|
|
while i < len(s) and s[i] != '"':
|
|
cur.append(s[i])
|
|
i += 1
|
|
if i == len(s):
|
|
errors.append("Unterminated quote in configuration comment")
|
|
cur.clear()
|
|
else:
|
|
cur.append(s[i])
|
|
i += 1
|
|
if cur:
|
|
parts.append(''.join(cur).strip())
|
|
|
|
return parts, errors
|
|
|
|
|
|
def mypy_comments_to_config_map(line: str,
|
|
template: Options) -> Tuple[Dict[str, str], List[str]]:
|
|
"""Rewrite the mypy comment syntax into ini file syntax.
|
|
|
|
Returns
|
|
"""
|
|
options = {}
|
|
entries, errors = split_directive(line)
|
|
for entry in entries:
|
|
if '=' not in entry:
|
|
name = entry
|
|
value = None
|
|
else:
|
|
name, value = (x.strip() for x in entry.split('=', 1))
|
|
|
|
name = name.replace('-', '_')
|
|
if value is None:
|
|
value = 'True'
|
|
options[name] = value
|
|
|
|
return options, errors
|
|
|
|
|
|
def parse_mypy_comments(
|
|
args: List[Tuple[int, str]],
|
|
template: Options) -> Tuple[Dict[str, object], List[Tuple[int, str]]]:
|
|
"""Parse a collection of inline mypy: configuration comments.
|
|
|
|
Returns a dictionary of options to be applied and a list of error messages
|
|
generated.
|
|
"""
|
|
|
|
errors: List[Tuple[int, str]] = []
|
|
sections = {}
|
|
|
|
for lineno, line in args:
|
|
# In order to easily match the behavior for bools, we abuse configparser.
|
|
# Oddly, the only way to get the SectionProxy object with the getboolean
|
|
# method is to create a config parser.
|
|
parser = configparser.RawConfigParser()
|
|
options, parse_errors = mypy_comments_to_config_map(line, template)
|
|
parser['dummy'] = options
|
|
errors.extend((lineno, x) for x in parse_errors)
|
|
|
|
stderr = StringIO()
|
|
strict_found = False
|
|
|
|
def set_strict_flags() -> None:
|
|
nonlocal strict_found
|
|
strict_found = True
|
|
|
|
new_sections, reports = parse_section(
|
|
'', template, set_strict_flags, parser['dummy'], ini_config_types, stderr=stderr)
|
|
errors.extend((lineno, x) for x in stderr.getvalue().strip().split('\n') if x)
|
|
if reports:
|
|
errors.append((lineno, "Reports not supported in inline configuration"))
|
|
if strict_found:
|
|
errors.append((lineno,
|
|
'Setting "strict" not supported in inline configuration: specify it in '
|
|
'a configuration file instead, or set individual inline flags '
|
|
'(see "mypy -h" for the list of flags enabled in strict mode)'))
|
|
|
|
sections.update(new_sections)
|
|
|
|
return sections, errors
|
|
|
|
|
|
def get_config_module_names(filename: Optional[str], modules: List[str]) -> str:
|
|
if not filename or not modules:
|
|
return ''
|
|
|
|
if not is_toml(filename):
|
|
return ", ".join(f"[mypy-{module}]" for module in modules)
|
|
|
|
return "module = ['%s']" % ("', '".join(sorted(modules)))
|
|
|
|
|
|
class ConfigTOMLValueError(ValueError):
|
|
pass
|