[GR-69732] More natural Java subclassing in Python. Github #470.

PullRequest: graalpython/3650

Author: timfel

CHANGELOG.md

@@ -8,6 +8,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * Allocation reporting via Truffle has been removed. Python object sizes were never reported correctly, so the data was misleading and there was a non-neglible overhead for object allocations even when reporting was inactive.
 * Better `readline` support via JLine. Autocompletion and history now works in `pdb`
 * Remove the intrinsified _ctypes module in favor of the native CPython version. This makes GraalPy's ctypes implementation more compatible and reduces the memory footprint of using ctypes.
+* Add a new, more natural style of subclassing Java classes from Python by passing the `new_style=True` keyword. Multiple levels of inheritance are supported, and `super()` calls both in the constructor override via `__new__` as well as in Java method overrides work as expected.
 
 ## Version 25.0.1
 * Allow users to keep going on unsupported JDK/OS/ARCH combinations at their own risk by opting out of early failure using `-Dtruffle.UseFallbackRuntime=true`, `-Dpolyglot.engine.userResourceCache=/set/to/a/writeable/dir`, `-Dpolyglot.engine.allowUnsupportedPlatform=true`, and `-Dpolyglot.python.UnsupportedPlatformEmulates=[linux|macos|windows]` and `-Dorg.graalvm.python.resources.exclude=native.files`.

docs/user/Native-Extensions.md

@@ -50,5 +50,6 @@ The implementation also relies on `venv` to work, even if you are not using exte
 To support creating multiple GraalPy contexts that access native modules within the same JVM or Native Image, we need to isolate them from each other.
 The current strategy for this is to copy the libraries and modify them such that the dynamic library loader of the operating system will isolate them for us.
 To do this, all GraalPy contexts in the same process (not just those in the same engine!) must set the `python.IsolateNativeModules` option to `true`.
+You should test your applications thoroughly if you want to use this feature, as there are many possiblities for native code to sidestep the library isolation through other process-wide global state.
 
 For more details on this, see [our implementation details](https://github.com/oracle/graalpython/blob/master/docs/contributor/IMPLEMENTATION_DETAILS.md#c-extension-copying).

graalpython/com.oracle.graal.python.test/src/tests/test_interop.py

@@ -720,6 +720,13 @@ def test_super(self):
         super(list, l).remove(0) # ArrayList#remove(int index)
         assert l == [6]
 
+    def test_issubclass_isinstance(self):
+        from java.util import ArrayList, List
+        assert issubclass(ArrayList, List)
+        assert issubclass(ArrayList, ArrayList)
+        assert isinstance(ArrayList(), List)
+        assert isinstance(ArrayList(), ArrayList)
+
     def test_java_array(self):
         import java
         il = java.type("int[]")(20)
@@ -1409,6 +1416,68 @@ def test_foreign_repl(self):
         assert repr(Integer).startswith('<JavaClass[java.lang.Integer] at')
         assert repr(i) == '22'
 
+    def test_natural_subclassing(self):
+        from java.util.logging import Level
+
+        class PythonLevel(Level, new_style=True):
+            def __new__(cls, name="default name", level=2):
+                return super().__new__(cls, name, level)
+
+            def __init__(self, *args, **kwarg):
+                self.misc_value = 42
+
+            def getName(self):
+                return super().getName() + " from Python with super()"
+
+            def pythonName(self):
+                return f"PythonName for Level {self.intValue()} named {super().getName()}"
+
+            def callStaticFromPython(self, name):
+                return self.parse(name)
+
+        pl = PythonLevel()
+        assert issubclass(PythonLevel, Level)
+        assert issubclass(PythonLevel, PythonLevel)
+        assert isinstance(pl, PythonLevel)
+        assert isinstance(pl, Level)
+        assert pl.getName() == "default name from Python with super()"
+        assert pl.intValue() == 2
+        assert pl.misc_value == 42
+        del pl.misc_value
+        try:
+            pl.misc_value
+        except AttributeError:
+            pass
+        else:
+            assert False
+        pl.misc_value = 43
+        assert pl.misc_value == 43
+        assert pl.pythonName() == "PythonName for Level 2 named default name"
+        assert pl.callStaticFromPython("INFO").getName() == "INFO"
+        assert PythonLevel.parse("INFO").getName() == "INFO"
+
+        class PythonLevel2(PythonLevel):
+            def __new__(cls):
+                return super().__new__(cls, "deeper name")
+
+            def pythonName(self):
+                return super().pythonName() + " from subclass"
+
+            def getName(self):
+                return super().getName() + " from subclass"
+
+
+        pl = PythonLevel2()
+        assert issubclass(PythonLevel2, Level)
+        assert issubclass(PythonLevel2, PythonLevel2)
+        assert isinstance(pl, PythonLevel2)
+        assert isinstance(pl, Level)
+        assert pl.getName() == "deeper name from Python with super() from subclass"
+        assert pl.pythonName() == "PythonName for Level 2 named deeper name from subclass"
+        assert pl.callStaticFromPython("INFO").getName() == "INFO"
+        assert PythonLevel2.parse("INFO").getName() == "INFO"
+
+
     def test_jython_star_import(self):
         if __graalpython__.jython_emulation_enabled:
             g = {}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

@@ -2434,22 +2434,22 @@ public abstract static class BuildClassNode extends PythonVarargsBuiltinNode {
 
         // No BoundaryCallContext: calls only internal well-behaved Python code
         @TruffleBoundary
-        private static Object buildJavaClass(Object namespace, TruffleString name, Object base) {
+        private static Object buildJavaClass(Object namespace, TruffleString name, Object base, PKeyword[] keywords) {
             // uncached PythonContext get, since this code path is slow in any case
             Object module = PythonContext.get(null).lookupBuiltinModule(T___GRAALPYTHON__);
             Object buildFunction = PyObjectLookupAttr.executeUncached(module, T_BUILD_JAVA_CLASS);
-            return CallNode.executeUncached(buildFunction, namespace, name, base);
+            return CallNode.executeUncached(buildFunction, new Object[]{namespace, name, base}, keywords);
         }
 
         @InliningCutoff
         private static Object buildJavaClass(VirtualFrame frame, Node inliningTarget, PythonLanguage language, PFunction function, Object[] arguments,
-                        CallDispatchers.FunctionCachedInvokeNode invokeBody,
+                        PKeyword[] keywords, CallDispatchers.FunctionCachedInvokeNode invokeBody,
                         TruffleString name) {
             PDict ns = PFactory.createDict(language, new DynamicObjectStorage(language));
             Object[] args = PArguments.create(0);
             PArguments.setSpecialArgument(args, ns);
             invokeBody.execute(frame, inliningTarget, function, args);
-            return buildJavaClass(ns, name, arguments[1]);
+            return buildJavaClass(ns, name, arguments[1], keywords);
         }
 
         @Specialization
@@ -2492,7 +2492,7 @@ protected Object doItNonFunction(VirtualFrame frame, Object function, Object[] a
 
             if (arguments.length == 2 && env.isHostObject(arguments[1]) && env.asHostObject(arguments[1]) instanceof Class<?>) {
                 // we want to subclass a Java class
-                return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, invokeBody, name);
+                return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, keywords, invokeBody, name);
             }
 
             class InitializeBuildClass {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java

@@ -36,20 +36,25 @@
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
+import com.oracle.graal.python.lib.PyObjectGetIter;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.runtime.GilNode;
 import com.oracle.graal.python.runtime.object.PFactory;
+import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.dsl.Bind;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.dsl.Specialization;
+import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.interop.InteropLibrary;
 import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.library.CachedLibrary;
+import com.oracle.truffle.api.nodes.Node;
 
 /*
  * NOTE: We are not using IndirectCallContext here in this file
@@ -67,10 +72,25 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
     @Builtin(name = J___BASES__, minNumOfPositionalArgs = 1, isGetter = true, isSetter = false)
     @GenerateNodeFactory
     abstract static class BasesNode extends PythonUnaryBuiltinNode {
-        @Specialization
-        static Object getBases(Object self,
+        @Specialization(limit = "2")
+        static Object getBases(VirtualFrame frame, Object self,
+                        @Bind Node inliningTarget,
+                        @CachedLibrary("self") InteropLibrary lib,
+                        @Cached PyObjectGetIter getIter,
+                        @Cached SequenceStorageNodes.CreateStorageFromIteratorNode createStorageFromIteratorNode,
                         @Bind PythonLanguage language) {
-            return PFactory.createEmptyTuple(language);
+            if (lib.hasMetaParents(self)) {
+                try {
+                    Object parents = lib.getMetaParents(self);
+                    Object iterObj = getIter.execute(frame, inliningTarget, parents);
+                    SequenceStorage storage = createStorageFromIteratorNode.execute(frame, iterObj);
+                    return PFactory.createTuple(language, storage);
+                } catch (UnsupportedMessageException e) {
+                    throw CompilerDirectives.shouldNotReachHere(e);
+                }
+            } else {
+                return PFactory.createEmptyTuple(language);
+            }
         }
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java

@@ -222,6 +222,7 @@
 import com.oracle.truffle.api.CompilerDirectives.ValueType;
 import com.oracle.truffle.api.HostCompilerDirectives.InliningCutoff;
 import com.oracle.truffle.api.RootCallTarget;
+import com.oracle.truffle.api.TruffleLanguage.Env;
 import com.oracle.truffle.api.dsl.Bind;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Cached.Exclusive;
@@ -237,6 +238,8 @@
 import com.oracle.truffle.api.frame.Frame;
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.interop.InteropLibrary;
+import com.oracle.truffle.api.interop.UnknownIdentifierException;
+import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.library.CachedLibrary;
 import com.oracle.truffle.api.nodes.ControlFlowException;
 import com.oracle.truffle.api.nodes.Node;
@@ -1482,23 +1485,23 @@ static boolean doManaged(PythonManagedClass left, PythonManagedClass right) {
         }
 
         @Specialization
-        static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClassType right) {
+        static boolean doTypeType(PythonBuiltinClassType left, PythonBuiltinClassType right) {
             return left == right;
         }
 
         @Specialization
-        static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClass right) {
+        static boolean doTypeClass(PythonBuiltinClassType left, PythonBuiltinClass right) {
             return left == right.getType();
         }
 
         @Specialization
-        static boolean doManaged(PythonBuiltinClass left, PythonBuiltinClassType right) {
+        static boolean doClassType(PythonBuiltinClass left, PythonBuiltinClassType right) {
             return left.getType() == right;
         }
 
         @Specialization
         @InliningCutoff
-        static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbstractNativeObject right,
+        static boolean doNative(PythonAbstractNativeObject left, PythonAbstractNativeObject right,
                         @CachedLibrary(limit = "1") InteropLibrary lib) {
             if (left == right) {
                 return true;
@@ -1509,6 +1512,44 @@ static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbst
             return lib.isIdentical(left.getPtr(), right.getPtr(), lib);
         }
 
+        @Specialization(guards = {"isForeignObject(left)", "isForeignObject(right)"})
+        @InliningCutoff
+        static boolean doOther(Object left, Object right,
+                        @Bind PythonContext context,
+                        @CachedLibrary(limit = "2") InteropLibrary lib) {
+            if (lib.isMetaObject(left) && lib.isMetaObject(right)) {
+                if (left == right) {
+                    return true;
+                }
+                // *sigh*... Host classes have split personality with a "static" and a "class"
+                // side, and that affects identity comparisons. And they report their "class" sides
+                // as bases, but importing from Java gives you the "static" side.
+                Env env = context.getEnv();
+                if (env.isHostObject(left) && env.isHostObject(right)) {
+                    // the activation of isMemberReadable and later readMember serves as branch
+                    // profile
+                    boolean leftIsStatic = lib.isMemberReadable(left, "class");
+                    boolean rightIsStatic = lib.isMemberReadable(right, "class");
+                    if (leftIsStatic != rightIsStatic) {
+                        try {
+                            if (leftIsStatic) {
+                                left = lib.readMember(left, "class");
+                            } else {
+                                assert rightIsStatic;
+                                right = lib.readMember(right, "class");
+                            }
+                        } catch (UnsupportedMessageException | UnknownIdentifierException e) {
+                            throw CompilerDirectives.shouldNotReachHere(e);
+                        }
+                    }
+                }
+                if (lib.isIdentical(left, right, lib)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         @Fallback
         static boolean doOther(@SuppressWarnings("unused") Object left, @SuppressWarnings("unused") Object right) {
             return false;