475 lines
20 KiB
Python
475 lines
20 KiB
Python
"""Intermediate representation of classes."""
|
|
|
|
from typing import List, Optional, Set, Tuple, Dict, NamedTuple
|
|
from mypy.backports import OrderedDict
|
|
|
|
from mypyc.common import JsonDict
|
|
from mypyc.ir.ops import Value, DeserMaps
|
|
from mypyc.ir.rtypes import RType, RInstance, deserialize_type
|
|
from mypyc.ir.func_ir import FuncIR, FuncDecl, FuncSignature
|
|
from mypyc.namegen import NameGenerator, exported_name
|
|
from mypyc.common import PROPSET_PREFIX
|
|
|
|
|
|
# Some notes on the vtable layout: Each concrete class has a vtable
|
|
# that contains function pointers for its methods. So that subclasses
|
|
# may be efficiently used when their parent class is expected, the
|
|
# layout of child vtables must be an extension of their base class's
|
|
# vtable.
|
|
#
|
|
# This makes multiple inheritance tricky, since obviously we cannot be
|
|
# an extension of multiple parent classes. We solve this by requiring
|
|
# all but one parent to be "traits", which we can operate on in a
|
|
# somewhat less efficient way. For each trait implemented by a class,
|
|
# we generate a separate vtable for the methods in that trait.
|
|
# We then store an array of (trait type, trait vtable) pointers alongside
|
|
# a class's main vtable. When we want to call a trait method, we
|
|
# (at runtime!) search the array of trait vtables to find the correct one,
|
|
# then call through it.
|
|
# Trait vtables additionally need entries for attribute getters and setters,
|
|
# since they can't always be in the same location.
|
|
#
|
|
# To keep down the number of indirections necessary, we store the
|
|
# array of trait vtables in the memory *before* the class vtable, and
|
|
# search it backwards. (This is a trick we can only do once---there
|
|
# are only two directions to store data in---but I don't think we'll
|
|
# need it again.)
|
|
# There are some tricks we could try in the future to store the trait
|
|
# vtables inline in the trait table (which would cut down one indirection),
|
|
# but this seems good enough for now.
|
|
#
|
|
# As an example:
|
|
# Imagine that we have a class B that inherits from a concrete class A
|
|
# and traits T1 and T2, and that A has methods foo() and
|
|
# bar() and B overrides bar() with a more specific type.
|
|
# Then B's vtable will look something like:
|
|
#
|
|
# T1 type object
|
|
# ptr to B's T1 trait vtable
|
|
# T2 type object
|
|
# ptr to B's T2 trait vtable
|
|
# -> | A.foo
|
|
# | Glue function that converts between A.bar's type and B.bar
|
|
# B.bar
|
|
# B.baz
|
|
#
|
|
# The arrow points to the "start" of the vtable (what vtable pointers
|
|
# point to) and the bars indicate which parts correspond to the parent
|
|
# class A's vtable layout.
|
|
#
|
|
# Classes that allow interpreted code to subclass them also have a
|
|
# "shadow vtable" that contains implementations that delegate to
|
|
# making a pycall, so that overridden methods in interpreted children
|
|
# will be called. (A better strategy could dynamically generate these
|
|
# vtables based on which methods are overridden in the children.)
|
|
|
|
# Descriptions of method and attribute entries in class vtables.
|
|
# The 'cls' field is the class that the method/attr was defined in,
|
|
# which might be a parent class.
|
|
# The 'shadow_method', if present, contains the method that should be
|
|
# placed in the class's shadow vtable (if it has one).
|
|
|
|
VTableMethod = NamedTuple(
|
|
'VTableMethod', [('cls', 'ClassIR'),
|
|
('name', str),
|
|
('method', FuncIR),
|
|
('shadow_method', Optional[FuncIR])])
|
|
|
|
|
|
VTableEntries = List[VTableMethod]
|
|
|
|
|
|
class ClassIR:
|
|
"""Intermediate representation of a class.
|
|
|
|
This also describes the runtime structure of native instances.
|
|
"""
|
|
|
|
def __init__(self, name: str, module_name: str, is_trait: bool = False,
|
|
is_generated: bool = False, is_abstract: bool = False,
|
|
is_ext_class: bool = True) -> None:
|
|
self.name = name
|
|
self.module_name = module_name
|
|
self.is_trait = is_trait
|
|
self.is_generated = is_generated
|
|
self.is_abstract = is_abstract
|
|
self.is_ext_class = is_ext_class
|
|
# An augmented class has additional methods separate from what mypyc generates.
|
|
# Right now the only one is dataclasses.
|
|
self.is_augmented = False
|
|
# Does this inherit from a Python class?
|
|
self.inherits_python = False
|
|
# Do instances of this class have __dict__?
|
|
self.has_dict = False
|
|
# Do we allow interpreted subclasses? Derived from a mypyc_attr.
|
|
self.allow_interpreted_subclasses = False
|
|
# Does this class need getseters to be generated for its attributes? (getseters are also
|
|
# added if is_generated is False)
|
|
self.needs_getseters = False
|
|
# Is this class declared as serializable (supports copy.copy
|
|
# and pickle) using @mypyc_attr(serializable=True)?
|
|
#
|
|
# Additionally, any class with this attribute False but with
|
|
# an __init__ that can be called without any arguments is
|
|
# *implicitly serializable*. In this case __init__ will be
|
|
# called during deserialization without arguments. If this is
|
|
# True, we match Python semantics and __init__ won't be called
|
|
# during deserialization.
|
|
#
|
|
# This impacts also all subclasses. Use is_serializable() to
|
|
# also consider base classes.
|
|
self._serializable = False
|
|
# If this a subclass of some built-in python class, the name
|
|
# of the object for that class. We currently only support this
|
|
# in a few ad-hoc cases.
|
|
self.builtin_base: Optional[str] = None
|
|
# Default empty constructor
|
|
self.ctor = FuncDecl(name, None, module_name, FuncSignature([], RInstance(self)))
|
|
|
|
self.attributes: OrderedDict[str, RType] = OrderedDict()
|
|
# Deletable attributes
|
|
self.deletable: List[str] = []
|
|
# We populate method_types with the signatures of every method before
|
|
# we generate methods, and we rely on this information being present.
|
|
self.method_decls: OrderedDict[str, FuncDecl] = OrderedDict()
|
|
# Map of methods that are actually present in an extension class
|
|
self.methods: OrderedDict[str, FuncIR] = OrderedDict()
|
|
# Glue methods for boxing/unboxing when a class changes the type
|
|
# while overriding a method. Maps from (parent class overridden, method)
|
|
# to IR of glue method.
|
|
self.glue_methods: Dict[Tuple[ClassIR, str], FuncIR] = OrderedDict()
|
|
|
|
# Properties are accessed like attributes, but have behavior like method calls.
|
|
# They don't belong in the methods dictionary, since we don't want to expose them to
|
|
# Python's method API. But we want to put them into our own vtable as methods, so that
|
|
# they are properly handled and overridden. The property dictionary values are a tuple
|
|
# containing a property getter and an optional property setter.
|
|
self.properties: OrderedDict[str, Tuple[FuncIR, Optional[FuncIR]]] = OrderedDict()
|
|
# We generate these in prepare_class_def so that we have access to them when generating
|
|
# other methods and properties that rely on these types.
|
|
self.property_types: OrderedDict[str, RType] = OrderedDict()
|
|
|
|
self.vtable: Optional[Dict[str, int]] = None
|
|
self.vtable_entries: VTableEntries = []
|
|
self.trait_vtables: OrderedDict[ClassIR, VTableEntries] = OrderedDict()
|
|
# N.B: base might not actually quite be the direct base.
|
|
# It is the nearest concrete base, but we allow a trait in between.
|
|
self.base: Optional[ClassIR] = None
|
|
self.traits: List[ClassIR] = []
|
|
# Supply a working mro for most generated classes. Real classes will need to
|
|
# fix it up.
|
|
self.mro: List[ClassIR] = [self]
|
|
# base_mro is the chain of concrete (non-trait) ancestors
|
|
self.base_mro: List[ClassIR] = [self]
|
|
|
|
# Direct subclasses of this class (use subclasses() to also include non-direct ones)
|
|
# None if separate compilation prevents this from working
|
|
self.children: Optional[List[ClassIR]] = []
|
|
|
|
# Instance attributes that are initialized in the class body.
|
|
self.attrs_with_defaults: Set[str] = set()
|
|
|
|
# Attributes that are always initialized in __init__ or class body
|
|
# (inferred in mypyc.analysis.attrdefined using interprocedural analysis)
|
|
self._always_initialized_attrs: Set[str] = set()
|
|
|
|
# Attributes that are sometimes initialized in __init__
|
|
self._sometimes_initialized_attrs: Set[str] = set()
|
|
|
|
# If True, __init__ can make 'self' visible to unanalyzed/arbitrary code
|
|
self.init_self_leak = False
|
|
|
|
def __repr__(self) -> str:
|
|
return (
|
|
"ClassIR("
|
|
"name={self.name}, module_name={self.module_name}, "
|
|
"is_trait={self.is_trait}, is_generated={self.is_generated}, "
|
|
"is_abstract={self.is_abstract}, is_ext_class={self.is_ext_class}"
|
|
")".format(self=self))
|
|
|
|
@property
|
|
def fullname(self) -> str:
|
|
return f"{self.module_name}.{self.name}"
|
|
|
|
def real_base(self) -> Optional['ClassIR']:
|
|
"""Return the actual concrete base class, if there is one."""
|
|
if len(self.mro) > 1 and not self.mro[1].is_trait:
|
|
return self.mro[1]
|
|
return None
|
|
|
|
def vtable_entry(self, name: str) -> int:
|
|
assert self.vtable is not None, "vtable not computed yet"
|
|
assert name in self.vtable, f'{self.name!r} has no attribute {name!r}'
|
|
return self.vtable[name]
|
|
|
|
def attr_details(self, name: str) -> Tuple[RType, 'ClassIR']:
|
|
for ir in self.mro:
|
|
if name in ir.attributes:
|
|
return ir.attributes[name], ir
|
|
if name in ir.property_types:
|
|
return ir.property_types[name], ir
|
|
raise KeyError(f'{self.name!r} has no attribute {name!r}')
|
|
|
|
def attr_type(self, name: str) -> RType:
|
|
return self.attr_details(name)[0]
|
|
|
|
def method_decl(self, name: str) -> FuncDecl:
|
|
for ir in self.mro:
|
|
if name in ir.method_decls:
|
|
return ir.method_decls[name]
|
|
raise KeyError(f'{self.name!r} has no attribute {name!r}')
|
|
|
|
def method_sig(self, name: str) -> FuncSignature:
|
|
return self.method_decl(name).sig
|
|
|
|
def has_method(self, name: str) -> bool:
|
|
try:
|
|
self.method_decl(name)
|
|
except KeyError:
|
|
return False
|
|
return True
|
|
|
|
def is_method_final(self, name: str) -> bool:
|
|
subs = self.subclasses()
|
|
if subs is None:
|
|
# TODO: Look at the final attribute!
|
|
return False
|
|
|
|
if self.has_method(name):
|
|
method_decl = self.method_decl(name)
|
|
for subc in subs:
|
|
if subc.method_decl(name) != method_decl:
|
|
return False
|
|
return True
|
|
else:
|
|
return not any(subc.has_method(name) for subc in subs)
|
|
|
|
def has_attr(self, name: str) -> bool:
|
|
try:
|
|
self.attr_type(name)
|
|
except KeyError:
|
|
return False
|
|
return True
|
|
|
|
def is_deletable(self, name: str) -> bool:
|
|
for ir in self.mro:
|
|
if name in ir.deletable:
|
|
return True
|
|
return False
|
|
|
|
def is_always_defined(self, name: str) -> bool:
|
|
if self.is_deletable(name):
|
|
return False
|
|
return name in self._always_initialized_attrs
|
|
|
|
def name_prefix(self, names: NameGenerator) -> str:
|
|
return names.private_name(self.module_name, self.name)
|
|
|
|
def struct_name(self, names: NameGenerator) -> str:
|
|
return f'{exported_name(self.fullname)}Object'
|
|
|
|
def get_method_and_class(self, name: str) -> Optional[Tuple[FuncIR, 'ClassIR']]:
|
|
for ir in self.mro:
|
|
if name in ir.methods:
|
|
return ir.methods[name], ir
|
|
|
|
return None
|
|
|
|
def get_method(self, name: str) -> Optional[FuncIR]:
|
|
res = self.get_method_and_class(name)
|
|
return res[0] if res else None
|
|
|
|
def subclasses(self) -> Optional[Set['ClassIR']]:
|
|
"""Return all subclasses of this class, both direct and indirect.
|
|
|
|
Return None if it is impossible to identify all subclasses, for example
|
|
because we are performing separate compilation.
|
|
"""
|
|
if self.children is None or self.allow_interpreted_subclasses:
|
|
return None
|
|
result = set(self.children)
|
|
for child in self.children:
|
|
if child.children:
|
|
child_subs = child.subclasses()
|
|
if child_subs is None:
|
|
return None
|
|
result.update(child_subs)
|
|
return result
|
|
|
|
def concrete_subclasses(self) -> Optional[List['ClassIR']]:
|
|
"""Return all concrete (i.e. non-trait and non-abstract) subclasses.
|
|
|
|
Include both direct and indirect subclasses. Place classes with no children first.
|
|
"""
|
|
subs = self.subclasses()
|
|
if subs is None:
|
|
return None
|
|
concrete = {c for c in subs if not (c.is_trait or c.is_abstract)}
|
|
# We place classes with no children first because they are more likely
|
|
# to appear in various isinstance() checks. We then sort leaves by name
|
|
# to get stable order.
|
|
return sorted(concrete, key=lambda c: (len(c.children or []), c.name))
|
|
|
|
def is_serializable(self) -> bool:
|
|
return any(ci._serializable for ci in self.mro)
|
|
|
|
def serialize(self) -> JsonDict:
|
|
return {
|
|
'name': self.name,
|
|
'module_name': self.module_name,
|
|
'is_trait': self.is_trait,
|
|
'is_ext_class': self.is_ext_class,
|
|
'is_abstract': self.is_abstract,
|
|
'is_generated': self.is_generated,
|
|
'is_augmented': self.is_augmented,
|
|
'inherits_python': self.inherits_python,
|
|
'has_dict': self.has_dict,
|
|
'allow_interpreted_subclasses': self.allow_interpreted_subclasses,
|
|
'needs_getseters': self.needs_getseters,
|
|
'_serializable': self._serializable,
|
|
'builtin_base': self.builtin_base,
|
|
'ctor': self.ctor.serialize(),
|
|
# We serialize dicts as lists to ensure order is preserved
|
|
'attributes': [(k, t.serialize()) for k, t in self.attributes.items()],
|
|
# We try to serialize a name reference, but if the decl isn't in methods
|
|
# then we can't be sure that will work so we serialize the whole decl.
|
|
'method_decls': [(k, d.id if k in self.methods else d.serialize())
|
|
for k, d in self.method_decls.items()],
|
|
# We serialize method fullnames out and put methods in a separate dict
|
|
'methods': [(k, m.id) for k, m in self.methods.items()],
|
|
'glue_methods': [
|
|
((cir.fullname, k), m.id)
|
|
for (cir, k), m in self.glue_methods.items()
|
|
],
|
|
|
|
# We serialize properties and property_types separately out of an
|
|
# abundance of caution about preserving dict ordering...
|
|
'property_types': [(k, t.serialize()) for k, t in self.property_types.items()],
|
|
'properties': list(self.properties),
|
|
|
|
'vtable': self.vtable,
|
|
'vtable_entries': serialize_vtable(self.vtable_entries),
|
|
'trait_vtables': [
|
|
(cir.fullname, serialize_vtable(v)) for cir, v in self.trait_vtables.items()
|
|
],
|
|
|
|
# References to class IRs are all just names
|
|
'base': self.base.fullname if self.base else None,
|
|
'traits': [cir.fullname for cir in self.traits],
|
|
'mro': [cir.fullname for cir in self.mro],
|
|
'base_mro': [cir.fullname for cir in self.base_mro],
|
|
'children': [
|
|
cir.fullname for cir in self.children
|
|
] if self.children is not None else None,
|
|
'deletable': self.deletable,
|
|
'attrs_with_defaults': sorted(self.attrs_with_defaults),
|
|
'_always_initialized_attrs': sorted(self._always_initialized_attrs),
|
|
'_sometimes_initialized_attrs': sorted(self._sometimes_initialized_attrs),
|
|
'init_self_leak': self.init_self_leak,
|
|
}
|
|
|
|
@classmethod
|
|
def deserialize(cls, data: JsonDict, ctx: 'DeserMaps') -> 'ClassIR':
|
|
fullname = data['module_name'] + '.' + data['name']
|
|
assert fullname in ctx.classes, "Class %s not in deser class map" % fullname
|
|
ir = ctx.classes[fullname]
|
|
|
|
ir.is_trait = data['is_trait']
|
|
ir.is_generated = data['is_generated']
|
|
ir.is_abstract = data['is_abstract']
|
|
ir.is_ext_class = data['is_ext_class']
|
|
ir.is_augmented = data['is_augmented']
|
|
ir.inherits_python = data['inherits_python']
|
|
ir.has_dict = data['has_dict']
|
|
ir.allow_interpreted_subclasses = data['allow_interpreted_subclasses']
|
|
ir.needs_getseters = data['needs_getseters']
|
|
ir._serializable = data['_serializable']
|
|
ir.builtin_base = data['builtin_base']
|
|
ir.ctor = FuncDecl.deserialize(data['ctor'], ctx)
|
|
ir.attributes = OrderedDict(
|
|
(k, deserialize_type(t, ctx)) for k, t in data['attributes']
|
|
)
|
|
ir.method_decls = OrderedDict((k, ctx.functions[v].decl
|
|
if isinstance(v, str) else FuncDecl.deserialize(v, ctx))
|
|
for k, v in data['method_decls'])
|
|
ir.methods = OrderedDict((k, ctx.functions[v]) for k, v in data['methods'])
|
|
ir.glue_methods = OrderedDict(
|
|
((ctx.classes[c], k), ctx.functions[v]) for (c, k), v in data['glue_methods']
|
|
)
|
|
ir.property_types = OrderedDict(
|
|
(k, deserialize_type(t, ctx)) for k, t in data['property_types']
|
|
)
|
|
ir.properties = OrderedDict(
|
|
(k, (ir.methods[k], ir.methods.get(PROPSET_PREFIX + k))) for k in data['properties']
|
|
)
|
|
|
|
ir.vtable = data['vtable']
|
|
ir.vtable_entries = deserialize_vtable(data['vtable_entries'], ctx)
|
|
ir.trait_vtables = OrderedDict(
|
|
(ctx.classes[k], deserialize_vtable(v, ctx)) for k, v in data['trait_vtables']
|
|
)
|
|
|
|
base = data['base']
|
|
ir.base = ctx.classes[base] if base else None
|
|
ir.traits = [ctx.classes[s] for s in data['traits']]
|
|
ir.mro = [ctx.classes[s] for s in data['mro']]
|
|
ir.base_mro = [ctx.classes[s] for s in data['base_mro']]
|
|
ir.children = data['children'] and [ctx.classes[s] for s in data['children']]
|
|
ir.deletable = data['deletable']
|
|
ir.attrs_with_defaults = set(data['attrs_with_defaults'])
|
|
ir._always_initialized_attrs = set(data['_always_initialized_attrs'])
|
|
ir._sometimes_initialized_attrs = set(data['_sometimes_initialized_attrs'])
|
|
ir.init_self_leak = data['init_self_leak']
|
|
|
|
return ir
|
|
|
|
|
|
class NonExtClassInfo:
|
|
"""Information needed to construct a non-extension class (Python class).
|
|
|
|
Includes the class dictionary, a tuple of base classes,
|
|
the class annotations dictionary, and the metaclass.
|
|
"""
|
|
|
|
def __init__(self, dict: Value, bases: Value, anns: Value, metaclass: Value) -> None:
|
|
self.dict = dict
|
|
self.bases = bases
|
|
self.anns = anns
|
|
self.metaclass = metaclass
|
|
|
|
|
|
def serialize_vtable_entry(entry: VTableMethod) -> JsonDict:
|
|
return {
|
|
'.class': 'VTableMethod',
|
|
'cls': entry.cls.fullname,
|
|
'name': entry.name,
|
|
'method': entry.method.decl.id,
|
|
'shadow_method': entry.shadow_method.decl.id if entry.shadow_method else None,
|
|
}
|
|
|
|
|
|
def serialize_vtable(vtable: VTableEntries) -> List[JsonDict]:
|
|
return [serialize_vtable_entry(v) for v in vtable]
|
|
|
|
|
|
def deserialize_vtable_entry(data: JsonDict, ctx: 'DeserMaps') -> VTableMethod:
|
|
if data['.class'] == 'VTableMethod':
|
|
return VTableMethod(
|
|
ctx.classes[data['cls']], data['name'], ctx.functions[data['method']],
|
|
ctx.functions[data['shadow_method']] if data['shadow_method'] else None)
|
|
assert False, "Bogus vtable .class: %s" % data['.class']
|
|
|
|
|
|
def deserialize_vtable(data: List[JsonDict], ctx: 'DeserMaps') -> VTableEntries:
|
|
return [deserialize_vtable_entry(x, ctx) for x in data]
|
|
|
|
|
|
def all_concrete_classes(class_ir: ClassIR) -> Optional[List[ClassIR]]:
|
|
"""Return all concrete classes among the class itself and its subclasses."""
|
|
concrete = class_ir.concrete_subclasses()
|
|
if concrete is None:
|
|
return None
|
|
if not (class_ir.is_abstract or class_ir.is_trait):
|
|
concrete.append(class_ir)
|
|
return concrete
|