441 lines
17 KiB
Python
441 lines
17 KiB
Python
|
from contextlib import contextmanager
|
||
|
|
from collections import defaultdict
|
||
|
|
|
||
|
|
from typing import Dict, List, Set, Iterator, Union, Optional, Tuple, cast
|
||
|
|
from typing_extensions import DefaultDict, TypeAlias as _TypeAlias
|
||
|
|
|
||
|
|
from mypy.types import (
|
||
|
|
Type, AnyType, PartialType, UnionType, TypeOfAny, NoneType, get_proper_type
|
||
|
|
)
|
||
|
|
from mypy.subtypes import is_subtype
|
||
|
|
from mypy.join import join_simple
|
||
|
|
from mypy.sametypes import is_same_type
|
||
|
|
from mypy.erasetype import remove_instance_last_known_values
|
||
|
|
from mypy.nodes import Expression, Var, RefExpr
|
||
|
|
from mypy.literals import Key, literal, literal_hash, subkeys
|
||
|
|
from mypy.nodes import IndexExpr, MemberExpr, AssignmentExpr, NameExpr
|
||
|
|
|
||
|
|
|
||
|
|
BindableExpression: _TypeAlias = Union[IndexExpr, MemberExpr, AssignmentExpr, NameExpr]
|
||
|
|
|
||
|
|
|
||
|
|
class Frame:
|
||
|
|
"""A Frame represents a specific point in the execution of a program.
|
||
|
|
It carries information about the current types of expressions at
|
||
|
|
that point, arising either from assignments to those expressions
|
||
|
|
or the result of isinstance checks. It also records whether it is
|
||
|
|
possible to reach that point at all.
|
||
|
|
|
||
|
|
This information is not copied into a new Frame when it is pushed
|
||
|
|
onto the stack, so a given Frame only has information about types
|
||
|
|
that were assigned in that frame.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, id: int, conditional_frame: bool = False) -> None:
|
||
|
|
self.id = id
|
||
|
|
self.types: Dict[Key, Type] = {}
|
||
|
|
self.unreachable = False
|
||
|
|
self.conditional_frame = conditional_frame
|
||
|
|
|
||
|
|
# Should be set only if we're entering a frame where it's not
|
||
|
|
# possible to accurately determine whether or not contained
|
||
|
|
# statements will be unreachable or not.
|
||
|
|
#
|
||
|
|
# Long-term, we should improve mypy to the point where we no longer
|
||
|
|
# need this field.
|
||
|
|
self.suppress_unreachable_warnings = False
|
||
|
|
|
||
|
|
|
||
|
|
Assigns = DefaultDict[Expression, List[Tuple[Type, Optional[Type]]]]
|
||
|
|
|
||
|
|
|
||
|
|
class ConditionalTypeBinder:
|
||
|
|
"""Keep track of conditional types of variables.
|
||
|
|
|
||
|
|
NB: Variables are tracked by literal expression, so it is possible
|
||
|
|
to confuse the binder; for example,
|
||
|
|
|
||
|
|
```
|
||
|
|
class A:
|
||
|
|
a = None # type: Union[int, str]
|
||
|
|
x = A()
|
||
|
|
lst = [x]
|
||
|
|
reveal_type(x.a) # Union[int, str]
|
||
|
|
x.a = 1
|
||
|
|
reveal_type(x.a) # int
|
||
|
|
reveal_type(lst[0].a) # Union[int, str]
|
||
|
|
lst[0].a = 'a'
|
||
|
|
reveal_type(x.a) # int
|
||
|
|
reveal_type(lst[0].a) # str
|
||
|
|
```
|
||
|
|
"""
|
||
|
|
# Stored assignments for situations with tuple/list lvalue and rvalue of union type.
|
||
|
|
# This maps an expression to a list of bound types for every item in the union type.
|
||
|
|
type_assignments: Optional[Assigns] = None
|
||
|
|
|
||
|
|
def __init__(self) -> None:
|
||
|
|
self.next_id = 1
|
||
|
|
|
||
|
|
# The stack of frames currently used. These map
|
||
|
|
# literal_hash(expr) -- literals like 'foo.bar' --
|
||
|
|
# to types. The last element of this list is the
|
||
|
|
# top-most, current frame. Each earlier element
|
||
|
|
# records the state as of when that frame was last
|
||
|
|
# on top of the stack.
|
||
|
|
self.frames = [Frame(self._get_id())]
|
||
|
|
|
||
|
|
# For frames higher in the stack, we record the set of
|
||
|
|
# Frames that can escape there, either by falling off
|
||
|
|
# the end of the frame or by a loop control construct
|
||
|
|
# or raised exception. The last element of self.frames
|
||
|
|
# has no corresponding element in this list.
|
||
|
|
self.options_on_return: List[List[Frame]] = []
|
||
|
|
|
||
|
|
# Maps literal_hash(expr) to get_declaration(expr)
|
||
|
|
# for every expr stored in the binder
|
||
|
|
self.declarations: Dict[Key, Optional[Type]] = {}
|
||
|
|
# Set of other keys to invalidate if a key is changed, e.g. x -> {x.a, x[0]}
|
||
|
|
# Whenever a new key (e.g. x.a.b) is added, we update this
|
||
|
|
self.dependencies: Dict[Key, Set[Key]] = {}
|
||
|
|
|
||
|
|
# Whether the last pop changed the newly top frame on exit
|
||
|
|
self.last_pop_changed = False
|
||
|
|
|
||
|
|
self.try_frames: Set[int] = set()
|
||
|
|
self.break_frames: List[int] = []
|
||
|
|
self.continue_frames: List[int] = []
|
||
|
|
|
||
|
|
def _get_id(self) -> int:
|
||
|
|
self.next_id += 1
|
||
|
|
return self.next_id
|
||
|
|
|
||
|
|
def _add_dependencies(self, key: Key, value: Optional[Key] = None) -> None:
|
||
|
|
if value is None:
|
||
|
|
value = key
|
||
|
|
else:
|
||
|
|
self.dependencies.setdefault(key, set()).add(value)
|
||
|
|
for elt in subkeys(key):
|
||
|
|
self._add_dependencies(elt, value)
|
||
|
|
|
||
|
|
def push_frame(self, conditional_frame: bool = False) -> Frame:
|
||
|
|
"""Push a new frame into the binder."""
|
||
|
|
f = Frame(self._get_id(), conditional_frame)
|
||
|
|
self.frames.append(f)
|
||
|
|
self.options_on_return.append([])
|
||
|
|
return f
|
||
|
|
|
||
|
|
def _put(self, key: Key, type: Type, index: int = -1) -> None:
|
||
|
|
self.frames[index].types[key] = type
|
||
|
|
|
||
|
|
def _get(self, key: Key, index: int = -1) -> Optional[Type]:
|
||
|
|
if index < 0:
|
||
|
|
index += len(self.frames)
|
||
|
|
for i in range(index, -1, -1):
|
||
|
|
if key in self.frames[i].types:
|
||
|
|
return self.frames[i].types[key]
|
||
|
|
return None
|
||
|
|
|
||
|
|
def put(self, expr: Expression, typ: Type) -> None:
|
||
|
|
if not isinstance(expr, (IndexExpr, MemberExpr, AssignmentExpr, NameExpr)):
|
||
|
|
return
|
||
|
|
if not literal(expr):
|
||
|
|
return
|
||
|
|
key = literal_hash(expr)
|
||
|
|
assert key is not None, 'Internal error: binder tried to put non-literal'
|
||
|
|
if key not in self.declarations:
|
||
|
|
self.declarations[key] = get_declaration(expr)
|
||
|
|
self._add_dependencies(key)
|
||
|
|
self._put(key, typ)
|
||
|
|
|
||
|
|
def unreachable(self) -> None:
|
||
|
|
self.frames[-1].unreachable = True
|
||
|
|
|
||
|
|
def suppress_unreachable_warnings(self) -> None:
|
||
|
|
self.frames[-1].suppress_unreachable_warnings = True
|
||
|
|
|
||
|
|
def get(self, expr: Expression) -> Optional[Type]:
|
||
|
|
key = literal_hash(expr)
|
||
|
|
assert key is not None, 'Internal error: binder tried to get non-literal'
|
||
|
|
return self._get(key)
|
||
|
|
|
||
|
|
def is_unreachable(self) -> bool:
|
||
|
|
# TODO: Copy the value of unreachable into new frames to avoid
|
||
|
|
# this traversal on every statement?
|
||
|
|
return any(f.unreachable for f in self.frames)
|
||
|
|
|
||
|
|
def is_unreachable_warning_suppressed(self) -> bool:
|
||
|
|
# TODO: See todo in 'is_unreachable'
|
||
|
|
return any(f.suppress_unreachable_warnings for f in self.frames)
|
||
|
|
|
||
|
|
def cleanse(self, expr: Expression) -> None:
|
||
|
|
"""Remove all references to a Node from the binder."""
|
||
|
|
key = literal_hash(expr)
|
||
|
|
assert key is not None, 'Internal error: binder tried cleanse non-literal'
|
||
|
|
self._cleanse_key(key)
|
||
|
|
|
||
|
|
def _cleanse_key(self, key: Key) -> None:
|
||
|
|
"""Remove all references to a key from the binder."""
|
||
|
|
for frame in self.frames:
|
||
|
|
if key in frame.types:
|
||
|
|
del frame.types[key]
|
||
|
|
|
||
|
|
def update_from_options(self, frames: List[Frame]) -> bool:
|
||
|
|
"""Update the frame to reflect that each key will be updated
|
||
|
|
as in one of the frames. Return whether any item changes.
|
||
|
|
|
||
|
|
If a key is declared as AnyType, only update it if all the
|
||
|
|
options are the same.
|
||
|
|
"""
|
||
|
|
|
||
|
|
frames = [f for f in frames if not f.unreachable]
|
||
|
|
changed = False
|
||
|
|
keys = {key for f in frames for key in f.types}
|
||
|
|
|
||
|
|
for key in keys:
|
||
|
|
current_value = self._get(key)
|
||
|
|
resulting_values = [f.types.get(key, current_value) for f in frames]
|
||
|
|
if any(x is None for x in resulting_values):
|
||
|
|
# We didn't know anything about key before
|
||
|
|
# (current_value must be None), and we still don't
|
||
|
|
# know anything about key in at least one possible frame.
|
||
|
|
continue
|
||
|
|
|
||
|
|
type = resulting_values[0]
|
||
|
|
assert type is not None
|
||
|
|
declaration_type = get_proper_type(self.declarations.get(key))
|
||
|
|
if isinstance(declaration_type, AnyType):
|
||
|
|
# At this point resulting values can't contain None, see continue above
|
||
|
|
if not all(is_same_type(type, cast(Type, t)) for t in resulting_values[1:]):
|
||
|
|
type = AnyType(TypeOfAny.from_another_any, source_any=declaration_type)
|
||
|
|
else:
|
||
|
|
for other in resulting_values[1:]:
|
||
|
|
assert other is not None
|
||
|
|
type = join_simple(self.declarations[key], type, other)
|
||
|
|
if current_value is None or not is_same_type(type, current_value):
|
||
|
|
self._put(key, type)
|
||
|
|
changed = True
|
||
|
|
|
||
|
|
self.frames[-1].unreachable = not frames
|
||
|
|
|
||
|
|
return changed
|
||
|
|
|
||
|
|
def pop_frame(self, can_skip: bool, fall_through: int) -> Frame:
|
||
|
|
"""Pop a frame and return it.
|
||
|
|
|
||
|
|
See frame_context() for documentation of fall_through.
|
||
|
|
"""
|
||
|
|
|
||
|
|
if fall_through > 0:
|
||
|
|
self.allow_jump(-fall_through)
|
||
|
|
|
||
|
|
result = self.frames.pop()
|
||
|
|
options = self.options_on_return.pop()
|
||
|
|
|
||
|
|
if can_skip:
|
||
|
|
options.insert(0, self.frames[-1])
|
||
|
|
|
||
|
|
self.last_pop_changed = self.update_from_options(options)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def accumulate_type_assignments(self) -> 'Iterator[Assigns]':
|
||
|
|
"""Push a new map to collect assigned types in multiassign from union.
|
||
|
|
|
||
|
|
If this map is not None, actual binding is deferred until all items in
|
||
|
|
the union are processed (a union of collected items is later bound
|
||
|
|
manually by the caller).
|
||
|
|
"""
|
||
|
|
old_assignments = None
|
||
|
|
if self.type_assignments is not None:
|
||
|
|
old_assignments = self.type_assignments
|
||
|
|
self.type_assignments = defaultdict(list)
|
||
|
|
yield self.type_assignments
|
||
|
|
self.type_assignments = old_assignments
|
||
|
|
|
||
|
|
def assign_type(self, expr: Expression,
|
||
|
|
type: Type,
|
||
|
|
declared_type: Optional[Type],
|
||
|
|
restrict_any: bool = False) -> None:
|
||
|
|
# We should erase last known value in binder, because if we are using it,
|
||
|
|
# it means that the target is not final, and therefore can't hold a literal.
|
||
|
|
type = remove_instance_last_known_values(type)
|
||
|
|
|
||
|
|
type = get_proper_type(type)
|
||
|
|
declared_type = get_proper_type(declared_type)
|
||
|
|
|
||
|
|
if self.type_assignments is not None:
|
||
|
|
# We are in a multiassign from union, defer the actual binding,
|
||
|
|
# just collect the types.
|
||
|
|
self.type_assignments[expr].append((type, declared_type))
|
||
|
|
return
|
||
|
|
if not isinstance(expr, (IndexExpr, MemberExpr, NameExpr)):
|
||
|
|
return None
|
||
|
|
if not literal(expr):
|
||
|
|
return
|
||
|
|
self.invalidate_dependencies(expr)
|
||
|
|
|
||
|
|
if declared_type is None:
|
||
|
|
# Not sure why this happens. It seems to mainly happen in
|
||
|
|
# member initialization.
|
||
|
|
return
|
||
|
|
if not is_subtype(type, declared_type):
|
||
|
|
# Pretty sure this is only happens when there's a type error.
|
||
|
|
|
||
|
|
# Ideally this function wouldn't be called if the
|
||
|
|
# expression has a type error, though -- do other kinds of
|
||
|
|
# errors cause this function to get called at invalid
|
||
|
|
# times?
|
||
|
|
return
|
||
|
|
|
||
|
|
enclosing_type = get_proper_type(self.most_recent_enclosing_type(expr, type))
|
||
|
|
if isinstance(enclosing_type, AnyType) and not restrict_any:
|
||
|
|
# If x is Any and y is int, after x = y we do not infer that x is int.
|
||
|
|
# This could be changed.
|
||
|
|
# Instead, since we narrowed type from Any in a recent frame (probably an
|
||
|
|
# isinstance check), but now it is reassigned, we broaden back
|
||
|
|
# to Any (which is the most recent enclosing type)
|
||
|
|
self.put(expr, enclosing_type)
|
||
|
|
# As a special case, when assigning Any to a variable with a
|
||
|
|
# declared Optional type that has been narrowed to None,
|
||
|
|
# replace all the Nones in the declared Union type with Any.
|
||
|
|
# This overrides the normal behavior of ignoring Any assignments to variables
|
||
|
|
# in order to prevent false positives.
|
||
|
|
# (See discussion in #3526)
|
||
|
|
elif (isinstance(type, AnyType)
|
||
|
|
and isinstance(declared_type, UnionType)
|
||
|
|
and any(isinstance(get_proper_type(item), NoneType) for item in declared_type.items)
|
||
|
|
and isinstance(get_proper_type(self.most_recent_enclosing_type(expr, NoneType())),
|
||
|
|
NoneType)):
|
||
|
|
# Replace any Nones in the union type with Any
|
||
|
|
new_items = [type if isinstance(get_proper_type(item), NoneType) else item
|
||
|
|
for item in declared_type.items]
|
||
|
|
self.put(expr, UnionType(new_items))
|
||
|
|
elif (isinstance(type, AnyType)
|
||
|
|
and not (isinstance(declared_type, UnionType)
|
||
|
|
and any(isinstance(get_proper_type(item), AnyType)
|
||
|
|
for item in declared_type.items))):
|
||
|
|
# Assigning an Any value doesn't affect the type to avoid false negatives, unless
|
||
|
|
# there is an Any item in a declared union type.
|
||
|
|
self.put(expr, declared_type)
|
||
|
|
else:
|
||
|
|
self.put(expr, type)
|
||
|
|
|
||
|
|
for i in self.try_frames:
|
||
|
|
# XXX This should probably not copy the entire frame, but
|
||
|
|
# just copy this variable into a single stored frame.
|
||
|
|
self.allow_jump(i)
|
||
|
|
|
||
|
|
def invalidate_dependencies(self, expr: BindableExpression) -> None:
|
||
|
|
"""Invalidate knowledge of types that include expr, but not expr itself.
|
||
|
|
|
||
|
|
For example, when expr is foo.bar, invalidate foo.bar.baz.
|
||
|
|
|
||
|
|
It is overly conservative: it invalidates globally, including
|
||
|
|
in code paths unreachable from here.
|
||
|
|
"""
|
||
|
|
key = literal_hash(expr)
|
||
|
|
assert key is not None
|
||
|
|
for dep in self.dependencies.get(key, set()):
|
||
|
|
self._cleanse_key(dep)
|
||
|
|
|
||
|
|
def most_recent_enclosing_type(self, expr: BindableExpression, type: Type) -> Optional[Type]:
|
||
|
|
type = get_proper_type(type)
|
||
|
|
if isinstance(type, AnyType):
|
||
|
|
return get_declaration(expr)
|
||
|
|
key = literal_hash(expr)
|
||
|
|
assert key is not None
|
||
|
|
enclosers = ([get_declaration(expr)] +
|
||
|
|
[f.types[key] for f in self.frames
|
||
|
|
if key in f.types and is_subtype(type, f.types[key])])
|
||
|
|
return enclosers[-1]
|
||
|
|
|
||
|
|
def allow_jump(self, index: int) -> None:
|
||
|
|
# self.frames and self.options_on_return have different lengths
|
||
|
|
# so make sure the index is positive
|
||
|
|
if index < 0:
|
||
|
|
index += len(self.options_on_return)
|
||
|
|
frame = Frame(self._get_id())
|
||
|
|
for f in self.frames[index + 1:]:
|
||
|
|
frame.types.update(f.types)
|
||
|
|
if f.unreachable:
|
||
|
|
frame.unreachable = True
|
||
|
|
self.options_on_return[index].append(frame)
|
||
|
|
|
||
|
|
def handle_break(self) -> None:
|
||
|
|
self.allow_jump(self.break_frames[-1])
|
||
|
|
self.unreachable()
|
||
|
|
|
||
|
|
def handle_continue(self) -> None:
|
||
|
|
self.allow_jump(self.continue_frames[-1])
|
||
|
|
self.unreachable()
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def frame_context(self, *, can_skip: bool, fall_through: int = 1,
|
||
|
|
break_frame: int = 0, continue_frame: int = 0,
|
||
|
|
conditional_frame: bool = False,
|
||
|
|
try_frame: bool = False) -> Iterator[Frame]:
|
||
|
|
"""Return a context manager that pushes/pops frames on enter/exit.
|
||
|
|
|
||
|
|
If can_skip is True, control flow is allowed to bypass the
|
||
|
|
newly-created frame.
|
||
|
|
|
||
|
|
If fall_through > 0, then it will allow control flow that
|
||
|
|
falls off the end of the frame to escape to its ancestor
|
||
|
|
`fall_through` levels higher. Otherwise control flow ends
|
||
|
|
at the end of the frame.
|
||
|
|
|
||
|
|
If break_frame > 0, then 'break' statements within this frame
|
||
|
|
will jump out to the frame break_frame levels higher than the
|
||
|
|
frame created by this call to frame_context. Similarly for
|
||
|
|
continue_frame and 'continue' statements.
|
||
|
|
|
||
|
|
If try_frame is true, then execution is allowed to jump at any
|
||
|
|
point within the newly created frame (or its descendants) to
|
||
|
|
its parent (i.e., to the frame that was on top before this
|
||
|
|
call to frame_context).
|
||
|
|
|
||
|
|
After the context manager exits, self.last_pop_changed indicates
|
||
|
|
whether any types changed in the newly-topmost frame as a result
|
||
|
|
of popping this frame.
|
||
|
|
"""
|
||
|
|
assert len(self.frames) > 1
|
||
|
|
|
||
|
|
if break_frame:
|
||
|
|
self.break_frames.append(len(self.frames) - break_frame)
|
||
|
|
if continue_frame:
|
||
|
|
self.continue_frames.append(len(self.frames) - continue_frame)
|
||
|
|
if try_frame:
|
||
|
|
self.try_frames.add(len(self.frames) - 1)
|
||
|
|
|
||
|
|
new_frame = self.push_frame(conditional_frame)
|
||
|
|
if try_frame:
|
||
|
|
# An exception may occur immediately
|
||
|
|
self.allow_jump(-1)
|
||
|
|
yield new_frame
|
||
|
|
self.pop_frame(can_skip, fall_through)
|
||
|
|
|
||
|
|
if break_frame:
|
||
|
|
self.break_frames.pop()
|
||
|
|
if continue_frame:
|
||
|
|
self.continue_frames.pop()
|
||
|
|
if try_frame:
|
||
|
|
self.try_frames.remove(len(self.frames) - 1)
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def top_frame_context(self) -> Iterator[Frame]:
|
||
|
|
"""A variant of frame_context for use at the top level of
|
||
|
|
a namespace (module, function, or class).
|
||
|
|
"""
|
||
|
|
assert len(self.frames) == 1
|
||
|
|
yield self.push_frame()
|
||
|
|
self.pop_frame(True, 0)
|
||
|
|
|
||
|
|
|
||
|
|
def get_declaration(expr: BindableExpression) -> Optional[Type]:
|
||
|
|
if isinstance(expr, RefExpr) and isinstance(expr.node, Var):
|
||
|
|
type = get_proper_type(expr.node.type)
|
||
|
|
if not isinstance(type, PartialType):
|
||
|
|
return type
|
||
|
|
return None
|