feat(python): support inheritance-based interface implementation

gh-pages/content/user-guides/lib-user/language-specific/python.md

@@ -16,45 +16,50 @@ Traditionally, **Python** developers expect to be able to either *implicitly* im
 required members, or *explicitly* implement interfaces by simply adding the interface to their class' or interface's
 inheritance chain (and implementing all required members):
 
-!!! bug "Incorrect Use"
-
-    ```py hl_lines="3"
-    from jsii_dependency import IJsiiInterface
-
-    class MyNewClass(IJsiiInterface):
-        """ Traditional implementation of an interface in Python.
-
-        This will not work with interfaces defined by jsii modules, as this will
-        likely cause a metaclass conflict that the user cannot solve.
-        """
-
-        # Member implementations...
-
-        ...
-    ```
-
 The [jsii type system][jsii-type-system] however does not support *structural typing*, and interfaces must **always** be
 *explicitly* implemented. In order to correctly declare implementation of an interface from a *jsii module*, the
 following syntax is used:
 
-```py hl_lines="1 4"
-import jsii
+```py hl_lines="3"
 from jsii_dependency import IJsiiInterface
 
-@jsii.implements(IJsiiInterface)
-class MyNewClass():
-    """ A jsii-supported implementation of the `IJsiiInterface` interface
+class MyNewClass(IJsiiInterface):
+    """ Traditional implementation of an interface in Python.
 
-    This will correctly register the explicit interface implementation on the
-    type's metadata, and ensure instances get correctly serialized to and from
-    the jsii kernel.
+    In multiple inheritance scenarios, you may encouter a metaclass conflict
+    if one of the ancestor interfaces is from a library generated with jsii-pacmak
+    releases <= 1.52.1. In this case, you may need to use the "Legacy Style" (see
+    below) declaration with the affected interfaces.
     """
 
     # Member implementations...
 
     ...
 ```
 
+!!! info Legacy Style
+    When using libraries generated by `jsii-pacmak` version `1.52.1` and earlier, using the inheritance chain style
+    above may result in metaclass conflicts when performing multiple inheritance. In such cases, the implementation can
+    be performed using the `@jsii.implements` decorator instead.
+
+    ```py hl_lines="1 4"
+    import jsii
+    from jsii_dependency import IJsiiInterface
+
+    @jsii.implements(IJsiiInterface)
+    class MyNewClass():
+        """ A jsii-supported implementation of the `IJsiiInterface` interface
+
+        This will correctly register the explicit interface implementation on the
+        type's metadata, and ensure instances get correctly serialized to and from
+        the jsii kernel.
+        """
+
+        # Member implementations...
+
+        ...
+    ```
+
 [jsii-type-system]: ../../../specification/2-type-system.md
 
 ## Property Overrides

packages/@jsii/python-runtime/build-tools/gen.ts

@@ -3,6 +3,10 @@
 import { copyFileSync, readdirSync, statSync, writeFileSync } from 'fs';
 import { resolve } from 'path';
 
+// Ensure we use a version number that is coherent with Python specifics.
+import { TargetName } from 'jsii-pacmak/lib/targets';
+import { toReleaseVersion } from 'jsii-pacmak/lib/targets/version-utils';
+
 const EMBEDDED_SOURCE = resolve(__dirname, '..', '..', 'runtime', 'webpack');
 const EMBEDDED_INFO = resolve(__dirname, '..', '..', 'runtime', 'package.json');
 
@@ -13,7 +17,7 @@ writeFileSync(
   resolve(__dirname, '..', 'src', 'jsii', '_metadata.json'),
   JSON.stringify(
     {
-      version: data.version,
+      version: toReleaseVersion(data.version, TargetName.PYTHON),
       description: data.description,
       license: data.license,
       author: data.author.name,

packages/@jsii/python-runtime/mypy.ini

@@ -1,2 +1,13 @@
 [mypy]
+disallow_untyped_decorators = True
+pretty = True
+python_version = 3.6
+warn_unused_configs = True
+warn_unused_ignores = True
+warn_unreachable = True
+
+[mypy-setuptools]
+ignore_missing_imports = True
+
+[mypy-importlib_resources]
 ignore_missing_imports = True

packages/@jsii/python-runtime/src/jsii/_kernel/__init__.py

@@ -3,7 +3,7 @@
 import itertools
 from types import FunctionType, MethodType, BuiltinFunctionType, LambdaType
 
-from typing import Any, List, Optional, Type, Union
+from typing import Any, Callable, List, Optional, Type
 
 import functools
 
@@ -12,9 +12,7 @@
 
 from ..errors import JSIIError
 from .. import _reference_map
-from .._utils import Singleton
 from .providers import BaseProvider, ProcessProvider
-from .types import Callback
 from .types import (
     EnumRef,
     LoadRequest,
@@ -63,7 +61,7 @@ def _get_overides(klass: Type, obj: Any) -> List[Override]:
     # We need to inspect each item in the MRO, until we get to our Type, at that
     # point we'll bail, because those methods are not the overriden methods, but the
     # "real" methods.
-    jsii_name = getattr(klass, "__jsii_type__", "Object")
+    jsii_name = getattr(klass, "__jsii_class__", None) or "Object"
     jsii_classes = [
         next(
             (
@@ -124,7 +122,7 @@ def _get_overides(klass: Type, obj: Any) -> List[Override]:
     return overrides
 
 
-def _recursize_dereference(kernel, d):
+def _recursize_dereference(kernel: "Kernel", d):
     if isinstance(d, dict):
         return {k: _recursize_dereference(kernel, v) for k, v in d.items()}
     elif isinstance(d, list):
@@ -137,7 +135,7 @@ def _recursize_dereference(kernel, d):
         return d
 
 
-def _dereferenced(fn):
+def _dereferenced(fn: Callable[..., Any]) -> Callable[..., Any]:
     @functools.wraps(fn)
     def wrapped(kernel, *args, **kwargs):
         return _recursize_dereference(kernel, fn(kernel, *args, **kwargs))
@@ -147,7 +145,7 @@ def wrapped(kernel, *args, **kwargs):
 
 # We need to recurse through our data structure and look for anything that the JSII
 # doesn't natively handle. These items will be created as "Object" types in the JSII.
-def _make_reference_for_native(kernel, d):
+def _make_reference_for_native(kernel: "Kernel", d):
     if isinstance(d, dict):
         return {
             "$jsii.map": {
@@ -181,6 +179,12 @@ def _make_reference_for_native(kernel, d):
                     },
                 }
             }
+
+        if not hasattr(d, "__jsii_ref__"):
+            # This is a user-defined interface implementation. It hasn't been registered with the JS kernel just yet, so
+            # we need to create a JS proxy for this value before we can send it across the wire.
+            kernel.create(d.__class__, d)
+
         return d
 
     elif isinstance(d, (int, type(None), str, float, bool, datetime.datetime)):
@@ -201,7 +205,7 @@ def _make_reference_for_native(kernel, d):
         return d
 
 
-def _handle_callback(kernel, callback):
+def _handle_callback(kernel: "Kernel", callback):
     # need to handle get, set requests here as well as invoke requests
     if callback.invoke:
         obj = _reference_map.resolve_id(callback.invoke.objref.ref)
@@ -222,24 +226,23 @@ def _handle_callback(kernel, callback):
 
 
 def _callback_till_result(
-    kernel, response: Callback, response_type: Type[KernelResponse]
+    kernel: "Kernel", response: Callback, response_type: Type[KernelResponse]
 ) -> Any:
-    while isinstance(response, Callback):
+    current: Any = response
+    while isinstance(current, Callback):
         try:
-            result = _handle_callback(kernel, response)
+            result = _handle_callback(kernel, current)
         except Exception as exc:
-            response = kernel.sync_complete(
-                response.cbid, str(exc), None, response_type
-            )
+            current = kernel.sync_complete(current.cbid, str(exc), None, response_type)
         else:
-            response = kernel.sync_complete(response.cbid, None, result, response_type)
+            current = kernel.sync_complete(current.cbid, None, result, response_type)
 
-    if isinstance(response, InvokeResponse):
-        return response.result
-    elif isinstance(response, GetResponse):
-        return response.value
+    if isinstance(current, InvokeResponse):
+        return current.result
+    elif isinstance(current, GetResponse):
+        return current.value
     else:
-        return response
+        return current
 
 
 @attr.s(auto_attribs=True, frozen=True, slots=True)
@@ -289,7 +292,7 @@ def create(self, klass: Type, obj: Any, args: Optional[List[Any]] = None) -> Obj
 
         response = self.provider.create(
             CreateRequest(
-                fqn=klass.__jsii_type__ or "Object",
+                fqn=getattr(klass, "__jsii_class__", None) or "Object",
                 args=_make_reference_for_native(self, args),
                 overrides=_get_overides(klass, obj),
                 interfaces=[

packages/@jsii/python-runtime/src/jsii/_kernel/providers/process.py

@@ -13,10 +13,10 @@
 import tempfile
 import threading
 
-from typing import TYPE_CHECKING, Type, Union, Mapping, IO, Any, AnyStr, Optional
+from typing import Type, Union, Mapping, IO, Any, AnyStr, Optional
 
 import attr
-import cattr  # type: ignore
+import cattr
 import dateutil.parser
 
 import jsii._embedded.jsii

packages/@jsii/python-runtime/src/jsii/_runtime.py

@@ -1,15 +1,16 @@
 import abc
 import os
+import warnings
 
 import attr
 
-from typing import cast, Any, Callable, ClassVar, List, Optional, Mapping, Type, TypeVar
+from typing import cast, Any, Callable, List, Mapping, Optional, Type, TypeVar
+from typing_extensions import Protocol
 
 from . import _reference_map
 from ._compat import importlib_resources
 from ._kernel import Kernel
 from .python import _ClassPropertyMeta
-from ._kernel.types import ObjRef
 
 
 # Yea, a global here is kind of gross, however, there's not really a better way of
@@ -67,7 +68,7 @@ def __new__(
         # Since their parent class will have the __jsii_type__ variable defined, they
         # will as well anyways.
         if jsii_type is not None:
-            attrs["__jsii_type__"] = jsii_type
+            attrs["__jsii_class__"] = attrs["__jsii_type__"] = jsii_type
         # The declared type should NOT be inherited by subclasses. This way we can identify whether
         # an MRO entry corresponds to a possible overrides contributor or not.
         attrs["__jsii_declared_type__"] = jsii_type
@@ -82,14 +83,6 @@ def __new__(
 
         return cast("JSIIMeta", obj)
 
-    def __call__(cls: Type[Any], *args: Any, **kwargs) -> Any:
-        inst = super().__call__(*args, **kwargs)
-
-        # Register this instance with our reference map.
-        _reference_map.register_reference(inst)
-
-        return inst
-
 
 class JSIIAbstractClass(abc.ABCMeta, JSIIMeta):
     pass
@@ -133,7 +126,19 @@ def deco(fn):
 
 
 def implements(*interfaces: Type[Any]) -> Callable[[T], T]:
-    def deco(cls):
+    def deco(cls: T) -> T:
+        # In the past, interfaces were rendered as Protocols, so they could not
+        # be directly extended. The @jsii.implements annotation was created to
+        # register the nominal type relationship. Now, interfaces are rendered
+        # as fully abstract base classes, and they should simply be extended. We
+        # emit a warning when the legacy usage is detected.
+        for interface in interfaces:
+            if type(interface) is not type(Protocol):
+                warnings.warn(
+                    f"{interface.__name__} is no longer a Protocol. Use of @jsii.implements with it is deprecated. Move this interface to the class' inhertiance list of {cls.__name__} instead!",
+                    stacklevel=2,
+                )
+
         cls.__jsii_type__ = getattr(cls, "__jsii_type__", None)
         cls.__jsii_ifaces__ = getattr(cls, "__jsii_ifaces__", []) + list(interfaces)
         return cls
@@ -143,14 +148,18 @@ def deco(cls):
 
 def interface(*, jsii_type: str) -> Callable[[T], T]:
     def deco(iface):
+        # Un-set __jsii_class__ as this is an interface, and not a class.
+        iface.__jsii_class__ = None
         iface.__jsii_type__ = jsii_type
+        # This interface "implements itself" - this is a trick to ease up implementation discovery.
+        iface.__jsii_ifaces__ = [iface] + getattr(iface, "__jsii_ifaces__", [])
         _reference_map.register_interface(iface)
         return iface
 
     return deco
 
 
-def proxy_for(abstract_class: Type[Any]) -> Type[Any]:
+def proxy_for(abstract_class: T) -> T:
     if not hasattr(abstract_class, "__jsii_proxy_class__"):
         raise TypeError(f"{abstract_class} is not a JSII Abstract class.")
 

packages/@jsii/python-runtime/src/jsii/_utils.py

@@ -1,6 +1,6 @@
 import functools
 
-from typing import Any, Mapping, Type
+from typing import Any, Callable, List, Mapping, Type, TypeVar
 
 
 class Singleton(type):
@@ -14,8 +14,11 @@ def __call__(cls, *args, **kwargs):
         return cls._instances[cls]
 
 
-def memoized_property(fgetter):
-    stored = []
+T = TypeVar("T", bound=Any)
+
+
+def memoized_property(fgetter: Callable[[Any], T]) -> property:
+    stored: List[T] = []
 
     @functools.wraps(fgetter)
     def wrapped(self):