diff --git a/src/runtime/Codecs/PyObjectConversions.cs b/src/runtime/Codecs/PyObjectConversions.cs index cde97c8c9..c7659c5d2 100644 --- a/src/runtime/Codecs/PyObjectConversions.cs +++ b/src/runtime/Codecs/PyObjectConversions.cs @@ -13,6 +13,10 @@ namespace Python.Runtime /// This class allows to register additional marshalling codecs. /// Python.NET will pick suitable encoder/decoder registered first /// + /// + /// Codecs' CanDecode and CanEncode run outside the registry lock, because a + /// codec may run Python code, which can hand the GIL to a thread already waiting on that lock. + /// public static class PyObjectConversions { static readonly DecoderGroup decoders = new(); @@ -29,6 +33,7 @@ public static void RegisterEncoder(IPyObjectEncoder encoder) lock (encoders) { encoders.Add(encoder); + clrToPython.Clear(); } } @@ -43,6 +48,7 @@ public static void RegisterDecoder(IPyObjectDecoder decoder) lock (decoders) { decoders.Add(decoder); + pythonToClr.Clear(); } } @@ -65,10 +71,12 @@ static readonly ConcurrentDictionary clrToPython = new(); static IPyObjectEncoder[] GetEncoders(Type type) { + IPyObjectEncoder[] registered; lock (encoders) { - return encoders.GetEncoders(type).ToArray(); + registered = encoders.ToArray(); } + return registered.SelectMany(encoder => encoder.GetEncoders(type)).ToArray(); } #endregion @@ -93,12 +101,15 @@ internal static bool TryDecode(BorrowedReference pyHandle, BorrowedReference pyT Debug.Assert(PyType.IsType(sourceTypeRef)); var pyType = new PyType(sourceTypeRef, prevalidated: true); - IPyObjectDecoder? decoder; + IPyObjectDecoder[] registered; lock (decoders) { - decoder = decoders.GetDecoder(pyType, targetType); - if (decoder == null) return default; + registered = decoders.ToArray(); } + IPyObjectDecoder? decoder = registered + .Select(candidate => candidate.GetDecoder(pyType, targetType)) + .FirstOrDefault(found => found != null); + if (decoder == null) return default; var decode = genericDecode.MakeGenericMethod(targetType)!; diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 412f3b711..986944900 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -65,6 +65,9 @@ static Converter() if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyBaseObjectType) + return objectType; + return null; } @@ -273,6 +276,63 @@ internal static bool ToManaged(BorrowedReference value, Type type, return Converter.ToManagedValue(value, type, out result, setError); } + [ThreadStatic] + private static int truthyBooleanScopes; + + /// + /// Like , but a + /// or target also accepts any Python + /// object and converts it by truthiness, as IronPython does. + /// + /// + /// Used where the target type is fixed: property and field assignment and delegate + /// return values. Method arguments use it only after no overload matched strictly, so a + /// overload never wins over an overload that takes the value itself. + /// + internal static bool ToManagedTruthy(BorrowedReference value, Type type, + out object? result, bool setError) + { + using (AllowTruthyBooleans()) + { + return ToManaged(value, type, out result, setError); + } + } + + /// + /// Lets targets accept any Python object by truthiness on this thread + /// until the returned scope is disposed. + /// + internal static TruthyBooleanScope AllowTruthyBooleans() + { + truthyBooleanScopes++; + return default; + } + + internal readonly struct TruthyBooleanScope : IDisposable + { + public void Dispose() => truthyBooleanScopes--; + } + + internal static bool TakesBoolean(IEnumerable methods) + { + foreach (MethodBase method in methods) + { + foreach (ParameterInfo parameter in method.GetParameters()) + { + Type type = parameter.ParameterType; + if (type.IsByRef) + { + type = type.GetElementType(); + } + if (type == boolType || type == typeof(bool?)) + { + return true; + } + } + } + return false; + } + internal static bool ToManagedValue(BorrowedReference value, Type obType, out object? result, bool setError) { @@ -612,6 +672,16 @@ internal static bool ToPrimitive(BorrowedReference value, Type obType, out objec result = false; return true; } + if (truthyBooleanScopes > 0) + { + int truth = Runtime.PyObject_IsTrue(value); + if (truth < 0) + { + goto convert_error; + } + result = truth != 0; + return true; + } if (setError) { goto type_error; diff --git a/src/runtime/DelegateManager.cs b/src/runtime/DelegateManager.cs index 4343b9ab7..ca7893155 100644 --- a/src/runtime/DelegateManager.cs +++ b/src/runtime/DelegateManager.cs @@ -295,7 +295,7 @@ protected Dispatcher(PyObject target, Type dtype) return null; } BorrowedReference item0 = Runtime.PyTuple_GetItem(op, 0); - if (!Converter.ToManaged(item0, rtype, out object? result0, true)) + if (!Converter.ToManagedTruthy(item0, rtype, out object? result0, true)) { Exceptions.RaiseTypeError($"The Python function returned a tuple where element 0 was not {rtype} (the return type)"); throw PythonException.ThrowLastAsClrException(); @@ -331,7 +331,7 @@ protected Dispatcher(PyObject target, Type dtype) return null; } - if (!Converter.ToManaged(op, rtype, out object? result, true)) + if (!Converter.ToManagedTruthy(op, rtype, out object? result, true)) { throw PythonException.ThrowLastAsClrException(); } diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 07ed4fe22..7578d2bf1 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -363,7 +363,21 @@ public MismatchedMethod(Exception exception, MethodBase mb) _methods = GetMethods(); } - return Bind(inst, args, kwargDict, _methods, matchGenerics: true); + Binding? binding = Bind(inst, args, kwargDict, _methods, matchGenerics: true); + if (binding is null && Converter.TakesBoolean(_methods)) + { + PythonException? strictError = PythonException.FetchCurrentOrNullRaw(); + using (Converter.AllowTruthyBooleans()) + { + binding = Bind(inst, args, kwargDict, _methods, matchGenerics: true); + } + if (binding is null && strictError is not null) + { + Exceptions.Clear(); + strictError.Restore(); + } + } + return binding; } static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics) diff --git a/src/runtime/Mixins/CollectionMixinsProvider.cs b/src/runtime/Mixins/CollectionMixinsProvider.cs index d1b19e4d8..05adbdac6 100644 --- a/src/runtime/Mixins/CollectionMixinsProvider.cs +++ b/src/runtime/Mixins/CollectionMixinsProvider.cs @@ -63,6 +63,11 @@ public IEnumerable GetBaseTypes(Type type, IList existingBases) newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin"))); } + if (interfaces.Contains(typeof(IDisposable))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("DisposableMixin"))); + } + if (newBases.Count == existingBases.Count) { return existingBases; diff --git a/src/runtime/Mixins/collections.py b/src/runtime/Mixins/collections.py index 95a6d8162..bbe19daf1 100644 --- a/src/runtime/Mixins/collections.py +++ b/src/runtime/Mixins/collections.py @@ -16,6 +16,18 @@ def close(self): class IterableMixin(col.Iterable): pass +class DisposableMixin: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if hasattr(self, 'Dispose'): + self.Dispose() + else: + from System import IDisposable + IDisposable(self).Dispose() + return False + class SizedMixin(col.Sized): def __len__(self): return self.Count diff --git a/src/runtime/Runtime.Delegates.cs b/src/runtime/Runtime.Delegates.cs index 6490c3fe5..ef92a14c0 100644 --- a/src/runtime/Runtime.Delegates.cs +++ b/src/runtime/Runtime.Delegates.cs @@ -227,6 +227,8 @@ static Delegates() _PyType_Lookup = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll)); PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll)); PyObject_GenericGetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetDict), GetUnmanagedDll(PythonDLL)); + PyObject_GenericSetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetDict), GetUnmanagedDll(PythonDLL)); + PyDescr_NewGetSet = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDescr_NewGetSet), GetUnmanagedDll(PythonDLL)); PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); try @@ -534,6 +536,8 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericSetDict { get; } + internal static delegate* unmanaged[Cdecl] PyDescr_NewGetSet { get; } internal static delegate* unmanaged[Cdecl] PyType_GetSlot { get; } internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; } internal static delegate* unmanaged[Cdecl] _Py_NewReference { get; } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 4e1c6156a..ea207a1e3 100755 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1647,6 +1647,12 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); + internal static IntPtr PyObject_GenericGetDictAddress => (IntPtr)Delegates.PyObject_GenericGetDict; + + internal static IntPtr PyObject_GenericSetDictAddress => (IntPtr)Delegates.PyObject_GenericSetDict; + + internal static NewReference PyDescr_NewGetSet(BorrowedReference type, IntPtr getset) => Delegates.PyDescr_NewGetSet(type, getset); + internal static void PyObject_GC_Del(StolenReference ob) => Delegates.PyObject_GC_Del(ob); diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 559d5148e..a37365f96 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -374,7 +374,7 @@ static PyTuple GetBaseTypeTuple(Type clrType) return new PyTuple(bases); } - internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList interfaces, BorrowedReference dictRef) + internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList interfaces, BorrowedReference dictRef, string? defaultNamespace = null, string? definitionSite = null) { // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation @@ -388,7 +388,7 @@ internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase // the derived class can have class attributes __assembly__ and __module__ which // control the name of the assembly and module the new type is created in. object? assembly = null; - object? namespaceStr = null; + object? namespaceStr = defaultNamespace; using (var assemblyKey = new PyString("__assembly__")) { @@ -418,7 +418,8 @@ internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase return ReflectedClrType.CreateSubclass(py_base_type, interfaces, name, ns: (string?)namespaceStr, assembly: (string?)assembly, - dict: dictRef); + dict: dictRef, + definitionSite: definitionSite); } internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 592eefd55..683701f70 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Reflection.Emit; using System.Runtime.InteropServices; +using System.Text; using Python.Runtime.Native; @@ -33,6 +34,8 @@ internal class ClassDerivedObject : ClassObject { private static Dictionary assemblyBuilders; private static Dictionary, ModuleBuilder> moduleBuilders; + private static readonly Dictionary cache = new(); + private static readonly Dictionary> emittedTypeNames = new(); static ClassDerivedObject() { @@ -44,6 +47,8 @@ public static void Reset() { assemblyBuilders = new Dictionary(); moduleBuilders = new Dictionary, ModuleBuilder>(); + cache.Clear(); + emittedTypeNames.Clear(); } internal ClassDerivedObject(Type tp) : base(tp) @@ -142,13 +147,19 @@ internal static NewReference ToPython(IPythonDerivedType obj) /// methods overridden to call out to python if the associated python /// object has overridden the method. /// + /// + /// Where the Python class was defined, part of the cache key so classes that share a name + /// and shape but come from different sources never share a generated type; null when the + /// caller asks for the type by explicit namespace, which keeps its type shared by name. + /// internal static Type CreateDerivedType(string name, Type baseType, IList typeInterfaces, BorrowedReference py_dict, string? namespaceStr, string? assemblyName, - string moduleName = "Python.Runtime.Dynamic.dll") + string moduleName = "Python.Runtime.Dynamic.dll", + string? definitionSite = null) { // TODO: clean up if (null != namespaceStr) @@ -161,6 +172,12 @@ internal static Type CreateDerivedType(string name, assemblyName = "Python.Runtime.Dynamic"; } + var shapeKey = CreateShapeKey(name, baseType, typeInterfaces, py_dict, assemblyName, moduleName, definitionSite); + if (cache.TryGetValue(shapeKey, out Type cachedType)) + return cachedType; + + var typeName = ReserveTypeName(assemblyName, name, shapeKey); + ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); Type baseClass = baseType; @@ -176,7 +193,7 @@ internal static Type CreateDerivedType(string name, baseClass = typeof(object); } - TypeBuilder typeBuilder = moduleBuilder.DefineType(name, + TypeBuilder typeBuilder = moduleBuilder.DefineType(typeName, TypeAttributes.Public | TypeAttributes.Class, baseClass, interfaces.ToArray()); @@ -219,10 +236,20 @@ internal static Type CreateDerivedType(string name, // override any virtual not already overridden by the properties above // also override any interface method. - var methods = baseType.GetMethods().Concat(interfaces.SelectMany(x => x.GetMethods())); + var interfaceMethods = interfaces + .SelectMany(x => new[] { x }.Concat(x.GetInterfaces())) + .Distinct() + .SelectMany(x => x.GetMethods()); + var methods = baseType.GetMethods().Concat(interfaceMethods); var virtualMethods = new HashSet(); + var overriddenSignatures = new HashSet(StringComparer.Ordinal); foreach (MethodInfo method in methods) { + if (!overriddenSignatures.Add(MethodSignature(method))) + { + continue; + } + if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | method.Attributes.HasFlag(MethodAttributes.Final) // overriding generic virtual methods is not supported @@ -271,22 +298,25 @@ internal static Type CreateDerivedType(string name, } } - // add the destructor so the python object created in the constructor gets destroyed - MethodBuilder methodBuilder = typeBuilder.DefineMethod("Finalize", - MethodAttributes.Family | - MethodAttributes.Virtual | - MethodAttributes.HideBySig, - CallingConventions.Standard, - typeof(void), - Type.EmptyTypes); - ILGenerator il = methodBuilder.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); + if (!typeof(IPythonDerivedType).IsAssignableFrom(baseClass)) + { + // add the destructor so the python object created in the constructor gets destroyed + MethodBuilder methodBuilder = typeBuilder.DefineMethod("Finalize", + MethodAttributes.Family | + MethodAttributes.Virtual | + MethodAttributes.HideBySig, + CallingConventions.Standard, + typeof(void), + Type.EmptyTypes); + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); #pragma warning disable CS0618 // PythonDerivedType is for internal use only - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize))); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize))); #pragma warning restore CS0618 // PythonDerivedType is for internal use only - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); - il.Emit(OpCodes.Ret); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); + il.Emit(OpCodes.Ret); + } Type type = typeBuilder.CreateType(); @@ -294,12 +324,122 @@ internal static Type CreateDerivedType(string name, Assembly assembly = Assembly.GetAssembly(type); AssemblyManager.ScanAssembly(assembly); - // FIXME: assemblyBuilder not used - AssemblyBuilder assemblyBuilder = assemblyBuilders[assemblyName]; + cache[shapeKey] = type; return type; } + /// + /// Describes the explicit CLR members a Python class declares, so a redefinition that + /// adds, removes or retypes them gets a new generated type instead of a cached one. + /// + private static string ExplicitMemberSignature(BorrowedReference py_dict) + { + if (py_dict == null || !Runtime.PyDict_Check(py_dict)) + return string.Empty; + + var members = new List(); + using var dict = new PyDict(py_dict); + using var keys = dict.Keys(); + foreach (PyObject pyKey in keys) + { + using (pyKey) + using (var value = dict[pyKey]) + { + if (value.HasAttr("_clr_property_type_")) + { + using var propertyType = value.GetAttr("_clr_property_type_"); + string accessors = (HasTruthyAttr(value, "fget") ? "g" : "") + + (HasTruthyAttr(value, "fset") ? "s" : ""); + members.Add("P:" + pyKey + ":" + propertyType + ":" + accessors); + } + if (value.HasAttr("_clr_return_type_") && value.HasAttr("_clr_arg_types_")) + { + using var returnType = value.GetAttr("_clr_return_type_"); + using var argTypes = value.GetAttr("_clr_arg_types_"); + string methodName = pyKey.ToString()!; + if (value.HasAttr("_clr_method_name_")) + { + using var declaredName = value.GetAttr("_clr_method_name_"); + methodName = declaredName.ToString()!; + } + members.Add("M:" + methodName + ":" + returnType + ":" + argTypes); + } + } + } + members.Sort(StringComparer.Ordinal); + return string.Join("|", members); + } + + private static string StableHash(string text) + { + unchecked + { + uint hash = 2166136261; + foreach (char character in text) + { + hash ^= character; + hash *= 16777619; + } + return hash.ToString("x8"); + } + } + + private static bool HasTruthyAttr(PyObject value, string attributeName) + { + if (!value.HasAttr(attributeName)) + return false; + using var attribute = value.GetAttr(attributeName); + return attribute.IsTrue(); + } + + private static string CreateShapeKey( + string typeName, + Type baseType, + IEnumerable interfaces, + BorrowedReference py_dict, + string assemblyName, + string moduleName, + string? definitionSite) + { + var builder = new StringBuilder(); + builder.Append(assemblyName).Append('|').Append(moduleName).Append('|').Append(typeName); + builder.Append('|').Append(definitionSite); + builder.Append('|').Append(baseType.AssemblyQualifiedName); + foreach (Type interfaceType in interfaces) + builder.Append('|').Append(interfaceType.AssemblyQualifiedName); + builder.Append('|').Append(ExplicitMemberSignature(py_dict)); + return builder.ToString(); + } + + private static string MethodSignature(MethodInfo method) + { + var parameterTypes = method.GetParameters().Select(param => param.ParameterType.FullName ?? param.ParameterType.Name); + return method.ReturnType.FullName + " " + method.Name + "(" + string.Join(",", parameterTypes) + ")"; + } + + /// + /// Picks the CLR name for a new generated type. The first shape of a Python class keeps + /// its visible name so it stays importable from its namespace; later shapes with the same + /// name get a short hexadecimal hash suffix because an assembly cannot define a name twice. + /// + private static string ReserveTypeName(string assemblyName, string typeName, string shapeKey) + { + if (!emittedTypeNames.TryGetValue(assemblyName, out HashSet names)) + { + names = new HashSet(StringComparer.Ordinal); + emittedTypeNames[assemblyName] = names; + } + + string candidate = typeName; + string hashedName = typeName + "_" + StableHash(shapeKey); + for (int attempt = 1; names.Contains(candidate); attempt++) + candidate = attempt == 1 ? hashedName : hashedName + "_" + attempt; + + names.Add(candidate); + return candidate; + } + /// /// Add a constructor override that calls the python ctor after calling the base type constructor. /// diff --git a/src/runtime/Types/FieldObject.cs b/src/runtime/Types/FieldObject.cs index af772afe2..662f1c3d5 100644 --- a/src/runtime/Types/FieldObject.cs +++ b/src/runtime/Types/FieldObject.cs @@ -122,7 +122,7 @@ public static int tp_descr_set(BorrowedReference ds, BorrowedReference ob, Borro } } - if (!Converter.ToManaged(val, info.FieldType, out var newval, true)) + if (!Converter.ToManagedTruthy(val, info.FieldType, out var newval, true)) { return -1; } diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index 57fcaa232..f24575aab 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -66,6 +66,57 @@ internal static PyType RestoreRuntimeData(MetatypeState storage) return PyCLRMetaType; } + static string? ModuleNamespace(PyDict clsDict) + { + if (!clsDict.HasKey("__module__")) + { + return null; + } + using var module = clsDict["__module__"]; + return PyString.IsStringType(module) ? module.As() : null; + } + + /// + /// The __file__ of the globals the class statement runs in, so same-named classes + /// from different scripts, which all run as __main__, never share a generated type. + /// + static string? DefinitionSite() + { + BorrowedReference globals = Runtime.PyEval_GetGlobals(); + if (globals.IsNull) + { + return null; + } + BorrowedReference file = Runtime.PyDict_GetItemString(globals, "__file__"); + return !file.IsNull && Runtime.PyString_Check(file) ? Runtime.GetManagedString(file) : null; + } + + /// + /// Generates the managed type for a class that only implements .NET interfaces and + /// declares no __namespace__. Unlike an explicit request, a failure here must not + /// break the class statement, so it falls back to a plain Python type as before. + /// + /// + /// The type is shared only by classes of the same name and shape defined from the same + /// file, so re-running a script reuses it while another script's class never takes it over. + /// + static NewReference TryCreateInterfaceSubType(BorrowedReference name, ClassBase baseType, IList interfaces, PyDict clsDict) + { + try + { + NewReference derived = TypeManager.CreateSubType(name, baseType, interfaces, clsDict, ModuleNamespace(clsDict), DefinitionSite()); + if (!derived.IsNull()) + { + return derived; + } + } + catch (Exception) + { + } + Exceptions.Clear(); + return default; + } + /// /// Metatype __new__ implementation. This is called to create a new /// class / type when a reflected class is subclassed. @@ -120,7 +171,8 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, } } // if the base type count is 0, there might still be interfaces to implement. - if (baseTypes.Count == 0) + bool implementsInterfacesOnly = baseTypes.Count == 0; + if (implementsInterfacesOnly) { baseTypes.Add(new ClassBase(typeof(object))); } @@ -172,7 +224,15 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, using var clsDict = new PyDict(dict); if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) { - return TypeManager.CreateSubType(name, baseTypes[0], interfaces, clsDict); + return RunClassCreationHooks(TypeManager.CreateSubType(name, baseTypes[0], interfaces, clsDict), dict, kw); + } + if (implementsInterfacesOnly) + { + NewReference derived = TryCreateInterfaceSubType(name, baseTypes[0], interfaces, clsDict); + if (!derived.IsNull()) + { + return RunClassCreationHooks(derived, dict, kw); + } } } @@ -221,6 +281,57 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, } + /// + /// Finishes a class whose type was generated rather than built by type.__new__, by + /// running the class-creation protocol type.__new__ would: __set_name__ on + /// every namespace value that defines it, then the parent's __init_subclass__ with + /// the class keywords. + /// + /// Takes ownership of ; returns null with the Python error set on failure. + static NewReference RunClassCreationHooks(NewReference type, BorrowedReference namespaceDict, BorrowedReference kw) + { + if (type.IsNull()) + { + return type; + } + try + { + using var cls = new PyObject(type.Borrow()); + using var namespaceItems = new PyDict(namespaceDict); + using (var members = namespaceItems.Items()) + { + foreach (PyObject member in members) + { + using (member) + using (var memberName = member[0]) + using (var value = member[1]) + using (var valueType = value.GetPythonType()) + { + if (!valueType.HasAttr("__set_name__")) + { + continue; + } + using var setName = valueType.GetAttr("__set_name__"); + setName.Invoke(value, cls, memberName).Dispose(); + } + } + } + using var builtins = PyModule.Import("builtins"); + using var super = builtins.GetAttr("super"); + using var parent = super.Invoke(cls, cls); + using var initSubclass = parent.GetAttr("__init_subclass__"); + using var keywords = kw.IsNull ? null : new PyDict(kw); + initSubclass.Invoke(Array.Empty(), keywords).Dispose(); + return type; + } + catch (PythonException error) + { + type.Dispose(); + error.Restore(); + return default; + } + } + public static NewReference tp_alloc(BorrowedReference mt, nint n) => Runtime.PyType_GenericAlloc(mt, n); diff --git a/src/runtime/Types/PropertyObject.cs b/src/runtime/Types/PropertyObject.cs index f09d1696a..3049c826a 100644 --- a/src/runtime/Types/PropertyObject.cs +++ b/src/runtime/Types/PropertyObject.cs @@ -124,7 +124,7 @@ public static int tp_descr_set(BorrowedReference ds, BorrowedReference ob, Borro } - if (!Converter.ToManaged(val, info.PropertyType, out var newval, true)) + if (!Converter.ToManagedTruthy(val, info.PropertyType, out var newval, true)) { return -1; } diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index 3d0aa7e99..3dede7bc1 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.InteropServices; using System.Runtime.Serialization; using static Python.Runtime.PythonException; @@ -70,7 +71,7 @@ internal void Restore(ClassBase cb) internal static NewReference CreateSubclass(ClassBase baseClass, IList interfaces, string name, string? assembly, string? ns, - BorrowedReference dict) + BorrowedReference dict, string? definitionSite = null) { try { @@ -79,8 +80,10 @@ internal static NewReference CreateSubclass(ClassBase baseClass, IList int interfaces, dict, ns, - assembly); + assembly, + definitionSite: definitionSite); + ClassManager.cache.Remove(subType); var py_type = GetOrCreate(subType); // by default the class dict will have all the C# methods in it, but as this is a @@ -94,6 +97,10 @@ internal static NewReference CreateSubclass(ClassBase baseClass, IList int ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type)); ThrowIfIsNotZero(Runtime.PyDict_DelItemString(cls_dict, "__classcell__")); } + WrapAsClassMethod(cls_dict, "__init_subclass__"); + AddInstanceDictDescriptor(py_type, cls_dict); + WrapAsClassMethod(cls_dict, "__class_getitem__"); + Runtime.PyType_Modified(py_type); return new NewReference(py_type); } @@ -103,6 +110,57 @@ internal static NewReference CreateSubclass(ClassBase baseClass, IList int } } + /// + /// Makes a plain function stored under a classmethod, as + /// type.__new__ does for __init_subclass__ and __class_getitem__. + /// + static void WrapAsClassMethod(BorrowedReference classDict, string name) + { + BorrowedReference member = Runtime.PyDict_GetItemString(classDict, name); + if (member.IsNull) + { + return; + } + using var types = PyModule.Import("types"); + using var functionType = types.GetAttr("FunctionType"); + if (Runtime.PyObject_TYPE(member) != functionType.Reference) + { + return; + } + using var builtins = PyModule.Import("builtins"); + using var classMethod = builtins.GetAttr("classmethod"); + using var function = new PyObject(member); + using var wrapped = classMethod.Invoke(function); + ThrowIfIsNotZero(Runtime.PyDict_SetItemString(classDict, name, wrapped)); + } + + static IntPtr instanceDictGetSet; + + /// + /// Exposes the instance dictionary as __dict__, which type.__new__ would have + /// added, so vars() and descriptors such as functools.cached_property work. + /// + static void AddInstanceDictDescriptor(PyType type, BorrowedReference classDict) + { + if (Util.ReadInt32(type, TypeOffset.tp_dictoffset) <= 0 + || !Runtime.PyDict_GetItemString(classDict, "__dict__").IsNull) + { + return; + } + if (instanceDictGetSet == IntPtr.Zero) + { + IntPtr getSet = Marshal.AllocHGlobal(5 * IntPtr.Size); + Marshal.WriteIntPtr(getSet, 0, Marshal.StringToHGlobalAnsi("__dict__")); + Marshal.WriteIntPtr(getSet, IntPtr.Size, Runtime.PyObject_GenericGetDictAddress); + Marshal.WriteIntPtr(getSet, 2 * IntPtr.Size, Runtime.PyObject_GenericSetDictAddress); + Marshal.WriteIntPtr(getSet, 3 * IntPtr.Size, IntPtr.Zero); + Marshal.WriteIntPtr(getSet, 4 * IntPtr.Size, IntPtr.Zero); + instanceDictGetSet = getSet; + } + using var descriptor = Runtime.PyDescr_NewGetSet(type, instanceDictGetSet); + ThrowIfIsNotZero(Runtime.PyDict_SetItemString(classDict, "__dict__", descriptor.BorrowOrThrow())); + } + static ReflectedClrType AllocateClass(Type clrType) { string name = TypeManager.GetPythonTypeName(clrType); diff --git a/tests/test_subclass.py b/tests/test_subclass.py index c6ab7650f..111ac7af2 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -29,6 +29,22 @@ def bar(self, x, i): return InterfaceTestClass +def redefine_interface_test_class_fixture(subnamespace): + """Redefine InterfaceTestClass with the same CLR identity and new behaviour.""" + + class InterfaceTestClass(IInterfaceTest): + """class that implements the test interface""" + __namespace__ = "Python.Test." + subnamespace + + def foo(self): + return "InterfaceTestClass2" + + def bar(self, x, i): + return "/".join([x] * i) + + return InterfaceTestClass + + def interface_generic_class_fixture(subnamespace): class GenericInterfaceImpl(IGenericInterface[int]): @@ -146,6 +162,20 @@ def test_derived_class(): x = FunctionsTest.pass_through(ob) assert id(x) == id(ob) + +def test_redefine_class(): + """Classes with the same CLR identity keep working after redefinition.""" + InterfaceTestClass = interface_test_class_fixture(test_redefine_class.__name__) + ob = InterfaceTestClass() + assert ob.foo() == "InterfaceTestClass" + + InterfaceTestClass = redefine_interface_test_class_fixture( + test_redefine_class.__name__ + ) + ob = InterfaceTestClass() + assert ob.foo() == "InterfaceTestClass2" + + def test_broken_derived_class(): """Test python class derived from managed type with invalid namespace""" with pytest.raises(TypeError):