diff --git a/src/runtime/Codecs/PyObjectConversions.cs b/src/runtime/Codecs/PyObjectConversions.cs
index cde97c8c9..c7659c5d2 100644
--- a/src/runtime/Codecs/PyObjectConversions.cs
+++ b/src/runtime/Codecs/PyObjectConversions.cs
@@ -13,6 +13,10 @@ namespace Python.Runtime
/// This class allows to register additional marshalling codecs.
/// Python.NET will pick suitable encoder/decoder registered first
///
+ ///
+ /// Codecs' CanDecode and CanEncode run outside the registry lock, because a
+ /// codec may run Python code, which can hand the GIL to a thread already waiting on that lock.
+ ///
public static class PyObjectConversions
{
static readonly DecoderGroup decoders = new();
@@ -29,6 +33,7 @@ public static void RegisterEncoder(IPyObjectEncoder encoder)
lock (encoders)
{
encoders.Add(encoder);
+ clrToPython.Clear();
}
}
@@ -43,6 +48,7 @@ public static void RegisterDecoder(IPyObjectDecoder decoder)
lock (decoders)
{
decoders.Add(decoder);
+ pythonToClr.Clear();
}
}
@@ -65,10 +71,12 @@ static readonly ConcurrentDictionary
clrToPython = new();
static IPyObjectEncoder[] GetEncoders(Type type)
{
+ IPyObjectEncoder[] registered;
lock (encoders)
{
- return encoders.GetEncoders(type).ToArray();
+ registered = encoders.ToArray();
}
+ return registered.SelectMany(encoder => encoder.GetEncoders(type)).ToArray();
}
#endregion
@@ -93,12 +101,15 @@ internal static bool TryDecode(BorrowedReference pyHandle, BorrowedReference pyT
Debug.Assert(PyType.IsType(sourceTypeRef));
var pyType = new PyType(sourceTypeRef, prevalidated: true);
- IPyObjectDecoder? decoder;
+ IPyObjectDecoder[] registered;
lock (decoders)
{
- decoder = decoders.GetDecoder(pyType, targetType);
- if (decoder == null) return default;
+ registered = decoders.ToArray();
}
+ IPyObjectDecoder? decoder = registered
+ .Select(candidate => candidate.GetDecoder(pyType, targetType))
+ .FirstOrDefault(found => found != null);
+ if (decoder == null) return default;
var decode = genericDecode.MakeGenericMethod(targetType)!;
diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs
index 412f3b711..986944900 100644
--- a/src/runtime/Converter.cs
+++ b/src/runtime/Converter.cs
@@ -65,6 +65,9 @@ static Converter()
if (op == Runtime.PyBoolType)
return boolType;
+ if (op == Runtime.PyBaseObjectType)
+ return objectType;
+
return null;
}
@@ -273,6 +276,63 @@ internal static bool ToManaged(BorrowedReference value, Type type,
return Converter.ToManagedValue(value, type, out result, setError);
}
+ [ThreadStatic]
+ private static int truthyBooleanScopes;
+
+ ///
+ /// Like , but a
+ /// or target also accepts any Python
+ /// object and converts it by truthiness, as IronPython does.
+ ///
+ ///
+ /// Used where the target type is fixed: property and field assignment and delegate
+ /// return values. Method arguments use it only after no overload matched strictly, so a
+ /// overload never wins over an overload that takes the value itself.
+ ///
+ internal static bool ToManagedTruthy(BorrowedReference value, Type type,
+ out object? result, bool setError)
+ {
+ using (AllowTruthyBooleans())
+ {
+ return ToManaged(value, type, out result, setError);
+ }
+ }
+
+ ///
+ /// Lets targets accept any Python object by truthiness on this thread
+ /// until the returned scope is disposed.
+ ///
+ internal static TruthyBooleanScope AllowTruthyBooleans()
+ {
+ truthyBooleanScopes++;
+ return default;
+ }
+
+ internal readonly struct TruthyBooleanScope : IDisposable
+ {
+ public void Dispose() => truthyBooleanScopes--;
+ }
+
+ internal static bool TakesBoolean(IEnumerable methods)
+ {
+ foreach (MethodBase method in methods)
+ {
+ foreach (ParameterInfo parameter in method.GetParameters())
+ {
+ Type type = parameter.ParameterType;
+ if (type.IsByRef)
+ {
+ type = type.GetElementType();
+ }
+ if (type == boolType || type == typeof(bool?))
+ {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
internal static bool ToManagedValue(BorrowedReference value, Type obType,
out object? result, bool setError)
{
@@ -612,6 +672,16 @@ internal static bool ToPrimitive(BorrowedReference value, Type obType, out objec
result = false;
return true;
}
+ if (truthyBooleanScopes > 0)
+ {
+ int truth = Runtime.PyObject_IsTrue(value);
+ if (truth < 0)
+ {
+ goto convert_error;
+ }
+ result = truth != 0;
+ return true;
+ }
if (setError)
{
goto type_error;
diff --git a/src/runtime/DelegateManager.cs b/src/runtime/DelegateManager.cs
index 4343b9ab7..ca7893155 100644
--- a/src/runtime/DelegateManager.cs
+++ b/src/runtime/DelegateManager.cs
@@ -295,7 +295,7 @@ protected Dispatcher(PyObject target, Type dtype)
return null;
}
BorrowedReference item0 = Runtime.PyTuple_GetItem(op, 0);
- if (!Converter.ToManaged(item0, rtype, out object? result0, true))
+ if (!Converter.ToManagedTruthy(item0, rtype, out object? result0, true))
{
Exceptions.RaiseTypeError($"The Python function returned a tuple where element 0 was not {rtype} (the return type)");
throw PythonException.ThrowLastAsClrException();
@@ -331,7 +331,7 @@ protected Dispatcher(PyObject target, Type dtype)
return null;
}
- if (!Converter.ToManaged(op, rtype, out object? result, true))
+ if (!Converter.ToManagedTruthy(op, rtype, out object? result, true))
{
throw PythonException.ThrowLastAsClrException();
}
diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs
index 07ed4fe22..7578d2bf1 100644
--- a/src/runtime/MethodBinder.cs
+++ b/src/runtime/MethodBinder.cs
@@ -363,7 +363,21 @@ public MismatchedMethod(Exception exception, MethodBase mb)
_methods = GetMethods();
}
- return Bind(inst, args, kwargDict, _methods, matchGenerics: true);
+ Binding? binding = Bind(inst, args, kwargDict, _methods, matchGenerics: true);
+ if (binding is null && Converter.TakesBoolean(_methods))
+ {
+ PythonException? strictError = PythonException.FetchCurrentOrNullRaw();
+ using (Converter.AllowTruthyBooleans())
+ {
+ binding = Bind(inst, args, kwargDict, _methods, matchGenerics: true);
+ }
+ if (binding is null && strictError is not null)
+ {
+ Exceptions.Clear();
+ strictError.Restore();
+ }
+ }
+ return binding;
}
static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics)
diff --git a/src/runtime/Mixins/CollectionMixinsProvider.cs b/src/runtime/Mixins/CollectionMixinsProvider.cs
index d1b19e4d8..05adbdac6 100644
--- a/src/runtime/Mixins/CollectionMixinsProvider.cs
+++ b/src/runtime/Mixins/CollectionMixinsProvider.cs
@@ -63,6 +63,11 @@ public IEnumerable GetBaseTypes(Type type, IList existingBases)
newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin")));
}
+ if (interfaces.Contains(typeof(IDisposable)))
+ {
+ newBases.Add(new PyType(this.Mixins.GetAttr("DisposableMixin")));
+ }
+
if (newBases.Count == existingBases.Count)
{
return existingBases;
diff --git a/src/runtime/Mixins/collections.py b/src/runtime/Mixins/collections.py
index 95a6d8162..bbe19daf1 100644
--- a/src/runtime/Mixins/collections.py
+++ b/src/runtime/Mixins/collections.py
@@ -16,6 +16,18 @@ def close(self):
class IterableMixin(col.Iterable):
pass
+class DisposableMixin:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if hasattr(self, 'Dispose'):
+ self.Dispose()
+ else:
+ from System import IDisposable
+ IDisposable(self).Dispose()
+ return False
+
class SizedMixin(col.Sized):
def __len__(self): return self.Count
diff --git a/src/runtime/Runtime.Delegates.cs b/src/runtime/Runtime.Delegates.cs
index 6490c3fe5..ef92a14c0 100644
--- a/src/runtime/Runtime.Delegates.cs
+++ b/src/runtime/Runtime.Delegates.cs
@@ -227,6 +227,8 @@ static Delegates()
_PyType_Lookup = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll));
PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll));
PyObject_GenericGetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetDict), GetUnmanagedDll(PythonDLL));
+ PyObject_GenericSetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetDict), GetUnmanagedDll(PythonDLL));
+ PyDescr_NewGetSet = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDescr_NewGetSet), GetUnmanagedDll(PythonDLL));
PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll));
PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll));
try
@@ -534,6 +536,8 @@ static Delegates()
internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; }
internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; }
internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; }
+ internal static delegate* unmanaged[Cdecl] PyObject_GenericSetDict { get; }
+ internal static delegate* unmanaged[Cdecl] PyDescr_NewGetSet { get; }
internal static delegate* unmanaged[Cdecl] PyType_GetSlot { get; }
internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; }
internal static delegate* unmanaged[Cdecl] _Py_NewReference { get; }
diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs
index 4e1c6156a..ea207a1e3 100755
--- a/src/runtime/Runtime.cs
+++ b/src/runtime/Runtime.cs
@@ -1647,6 +1647,12 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe
internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero);
internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context);
+ internal static IntPtr PyObject_GenericGetDictAddress => (IntPtr)Delegates.PyObject_GenericGetDict;
+
+ internal static IntPtr PyObject_GenericSetDictAddress => (IntPtr)Delegates.PyObject_GenericSetDict;
+
+ internal static NewReference PyDescr_NewGetSet(BorrowedReference type, IntPtr getset) => Delegates.PyDescr_NewGetSet(type, getset);
+
internal static void PyObject_GC_Del(StolenReference ob) => Delegates.PyObject_GC_Del(ob);
diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs
index 559d5148e..a37365f96 100644
--- a/src/runtime/TypeManager.cs
+++ b/src/runtime/TypeManager.cs
@@ -374,7 +374,7 @@ static PyTuple GetBaseTypeTuple(Type clrType)
return new PyTuple(bases);
}
- internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList interfaces, BorrowedReference dictRef)
+ internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList interfaces, BorrowedReference dictRef, string? defaultNamespace = null, string? definitionSite = null)
{
// Utility to create a subtype of a managed type with the ability for the
// a python subtype able to override the managed implementation
@@ -388,7 +388,7 @@ internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase
// the derived class can have class attributes __assembly__ and __module__ which
// control the name of the assembly and module the new type is created in.
object? assembly = null;
- object? namespaceStr = null;
+ object? namespaceStr = defaultNamespace;
using (var assemblyKey = new PyString("__assembly__"))
{
@@ -418,7 +418,8 @@ internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase
return ReflectedClrType.CreateSubclass(py_base_type, interfaces, name,
ns: (string?)namespaceStr,
assembly: (string?)assembly,
- dict: dictRef);
+ dict: dictRef,
+ definitionSite: definitionSite);
}
internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc)
diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs
index 592eefd55..683701f70 100644
--- a/src/runtime/Types/ClassDerived.cs
+++ b/src/runtime/Types/ClassDerived.cs
@@ -6,6 +6,7 @@
using System.Reflection;
using System.Reflection.Emit;
using System.Runtime.InteropServices;
+using System.Text;
using Python.Runtime.Native;
@@ -33,6 +34,8 @@ internal class ClassDerivedObject : ClassObject
{
private static Dictionary assemblyBuilders;
private static Dictionary, ModuleBuilder> moduleBuilders;
+ private static readonly Dictionary cache = new();
+ private static readonly Dictionary> emittedTypeNames = new();
static ClassDerivedObject()
{
@@ -44,6 +47,8 @@ public static void Reset()
{
assemblyBuilders = new Dictionary();
moduleBuilders = new Dictionary, ModuleBuilder>();
+ cache.Clear();
+ emittedTypeNames.Clear();
}
internal ClassDerivedObject(Type tp) : base(tp)
@@ -142,13 +147,19 @@ internal static NewReference ToPython(IPythonDerivedType obj)
/// methods overridden to call out to python if the associated python
/// object has overridden the method.
///
+ ///
+ /// Where the Python class was defined, part of the cache key so classes that share a name
+ /// and shape but come from different sources never share a generated type; null when the
+ /// caller asks for the type by explicit namespace, which keeps its type shared by name.
+ ///
internal static Type CreateDerivedType(string name,
Type baseType,
IList typeInterfaces,
BorrowedReference py_dict,
string? namespaceStr,
string? assemblyName,
- string moduleName = "Python.Runtime.Dynamic.dll")
+ string moduleName = "Python.Runtime.Dynamic.dll",
+ string? definitionSite = null)
{
// TODO: clean up
if (null != namespaceStr)
@@ -161,6 +172,12 @@ internal static Type CreateDerivedType(string name,
assemblyName = "Python.Runtime.Dynamic";
}
+ var shapeKey = CreateShapeKey(name, baseType, typeInterfaces, py_dict, assemblyName, moduleName, definitionSite);
+ if (cache.TryGetValue(shapeKey, out Type cachedType))
+ return cachedType;
+
+ var typeName = ReserveTypeName(assemblyName, name, shapeKey);
+
ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName);
Type baseClass = baseType;
@@ -176,7 +193,7 @@ internal static Type CreateDerivedType(string name,
baseClass = typeof(object);
}
- TypeBuilder typeBuilder = moduleBuilder.DefineType(name,
+ TypeBuilder typeBuilder = moduleBuilder.DefineType(typeName,
TypeAttributes.Public | TypeAttributes.Class,
baseClass,
interfaces.ToArray());
@@ -219,10 +236,20 @@ internal static Type CreateDerivedType(string name,
// override any virtual not already overridden by the properties above
// also override any interface method.
- var methods = baseType.GetMethods().Concat(interfaces.SelectMany(x => x.GetMethods()));
+ var interfaceMethods = interfaces
+ .SelectMany(x => new[] { x }.Concat(x.GetInterfaces()))
+ .Distinct()
+ .SelectMany(x => x.GetMethods());
+ var methods = baseType.GetMethods().Concat(interfaceMethods);
var virtualMethods = new HashSet();
+ var overriddenSignatures = new HashSet(StringComparer.Ordinal);
foreach (MethodInfo method in methods)
{
+ if (!overriddenSignatures.Add(MethodSignature(method)))
+ {
+ continue;
+ }
+
if (!method.Attributes.HasFlag(MethodAttributes.Virtual) |
method.Attributes.HasFlag(MethodAttributes.Final)
// overriding generic virtual methods is not supported
@@ -271,22 +298,25 @@ internal static Type CreateDerivedType(string name,
}
}
- // add the destructor so the python object created in the constructor gets destroyed
- MethodBuilder methodBuilder = typeBuilder.DefineMethod("Finalize",
- MethodAttributes.Family |
- MethodAttributes.Virtual |
- MethodAttributes.HideBySig,
- CallingConventions.Standard,
- typeof(void),
- Type.EmptyTypes);
- ILGenerator il = methodBuilder.GetILGenerator();
- il.Emit(OpCodes.Ldarg_0);
+ if (!typeof(IPythonDerivedType).IsAssignableFrom(baseClass))
+ {
+ // add the destructor so the python object created in the constructor gets destroyed
+ MethodBuilder methodBuilder = typeBuilder.DefineMethod("Finalize",
+ MethodAttributes.Family |
+ MethodAttributes.Virtual |
+ MethodAttributes.HideBySig,
+ CallingConventions.Standard,
+ typeof(void),
+ Type.EmptyTypes);
+ ILGenerator il = methodBuilder.GetILGenerator();
+ il.Emit(OpCodes.Ldarg_0);
#pragma warning disable CS0618 // PythonDerivedType is for internal use only
- il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize)));
+ il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize)));
#pragma warning restore CS0618 // PythonDerivedType is for internal use only
- il.Emit(OpCodes.Ldarg_0);
- il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance));
- il.Emit(OpCodes.Ret);
+ il.Emit(OpCodes.Ldarg_0);
+ il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance));
+ il.Emit(OpCodes.Ret);
+ }
Type type = typeBuilder.CreateType();
@@ -294,12 +324,122 @@ internal static Type CreateDerivedType(string name,
Assembly assembly = Assembly.GetAssembly(type);
AssemblyManager.ScanAssembly(assembly);
- // FIXME: assemblyBuilder not used
- AssemblyBuilder assemblyBuilder = assemblyBuilders[assemblyName];
+ cache[shapeKey] = type;
return type;
}
+ ///
+ /// Describes the explicit CLR members a Python class declares, so a redefinition that
+ /// adds, removes or retypes them gets a new generated type instead of a cached one.
+ ///
+ private static string ExplicitMemberSignature(BorrowedReference py_dict)
+ {
+ if (py_dict == null || !Runtime.PyDict_Check(py_dict))
+ return string.Empty;
+
+ var members = new List();
+ using var dict = new PyDict(py_dict);
+ using var keys = dict.Keys();
+ foreach (PyObject pyKey in keys)
+ {
+ using (pyKey)
+ using (var value = dict[pyKey])
+ {
+ if (value.HasAttr("_clr_property_type_"))
+ {
+ using var propertyType = value.GetAttr("_clr_property_type_");
+ string accessors = (HasTruthyAttr(value, "fget") ? "g" : "")
+ + (HasTruthyAttr(value, "fset") ? "s" : "");
+ members.Add("P:" + pyKey + ":" + propertyType + ":" + accessors);
+ }
+ if (value.HasAttr("_clr_return_type_") && value.HasAttr("_clr_arg_types_"))
+ {
+ using var returnType = value.GetAttr("_clr_return_type_");
+ using var argTypes = value.GetAttr("_clr_arg_types_");
+ string methodName = pyKey.ToString()!;
+ if (value.HasAttr("_clr_method_name_"))
+ {
+ using var declaredName = value.GetAttr("_clr_method_name_");
+ methodName = declaredName.ToString()!;
+ }
+ members.Add("M:" + methodName + ":" + returnType + ":" + argTypes);
+ }
+ }
+ }
+ members.Sort(StringComparer.Ordinal);
+ return string.Join("|", members);
+ }
+
+ private static string StableHash(string text)
+ {
+ unchecked
+ {
+ uint hash = 2166136261;
+ foreach (char character in text)
+ {
+ hash ^= character;
+ hash *= 16777619;
+ }
+ return hash.ToString("x8");
+ }
+ }
+
+ private static bool HasTruthyAttr(PyObject value, string attributeName)
+ {
+ if (!value.HasAttr(attributeName))
+ return false;
+ using var attribute = value.GetAttr(attributeName);
+ return attribute.IsTrue();
+ }
+
+ private static string CreateShapeKey(
+ string typeName,
+ Type baseType,
+ IEnumerable interfaces,
+ BorrowedReference py_dict,
+ string assemblyName,
+ string moduleName,
+ string? definitionSite)
+ {
+ var builder = new StringBuilder();
+ builder.Append(assemblyName).Append('|').Append(moduleName).Append('|').Append(typeName);
+ builder.Append('|').Append(definitionSite);
+ builder.Append('|').Append(baseType.AssemblyQualifiedName);
+ foreach (Type interfaceType in interfaces)
+ builder.Append('|').Append(interfaceType.AssemblyQualifiedName);
+ builder.Append('|').Append(ExplicitMemberSignature(py_dict));
+ return builder.ToString();
+ }
+
+ private static string MethodSignature(MethodInfo method)
+ {
+ var parameterTypes = method.GetParameters().Select(param => param.ParameterType.FullName ?? param.ParameterType.Name);
+ return method.ReturnType.FullName + " " + method.Name + "(" + string.Join(",", parameterTypes) + ")";
+ }
+
+ ///
+ /// Picks the CLR name for a new generated type. The first shape of a Python class keeps
+ /// its visible name so it stays importable from its namespace; later shapes with the same
+ /// name get a short hexadecimal hash suffix because an assembly cannot define a name twice.
+ ///
+ private static string ReserveTypeName(string assemblyName, string typeName, string shapeKey)
+ {
+ if (!emittedTypeNames.TryGetValue(assemblyName, out HashSet names))
+ {
+ names = new HashSet(StringComparer.Ordinal);
+ emittedTypeNames[assemblyName] = names;
+ }
+
+ string candidate = typeName;
+ string hashedName = typeName + "_" + StableHash(shapeKey);
+ for (int attempt = 1; names.Contains(candidate); attempt++)
+ candidate = attempt == 1 ? hashedName : hashedName + "_" + attempt;
+
+ names.Add(candidate);
+ return candidate;
+ }
+
///
/// Add a constructor override that calls the python ctor after calling the base type constructor.
///
diff --git a/src/runtime/Types/FieldObject.cs b/src/runtime/Types/FieldObject.cs
index af772afe2..662f1c3d5 100644
--- a/src/runtime/Types/FieldObject.cs
+++ b/src/runtime/Types/FieldObject.cs
@@ -122,7 +122,7 @@ public static int tp_descr_set(BorrowedReference ds, BorrowedReference ob, Borro
}
}
- if (!Converter.ToManaged(val, info.FieldType, out var newval, true))
+ if (!Converter.ToManagedTruthy(val, info.FieldType, out var newval, true))
{
return -1;
}
diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs
index 57fcaa232..f24575aab 100644
--- a/src/runtime/Types/MetaType.cs
+++ b/src/runtime/Types/MetaType.cs
@@ -66,6 +66,57 @@ internal static PyType RestoreRuntimeData(MetatypeState storage)
return PyCLRMetaType;
}
+ static string? ModuleNamespace(PyDict clsDict)
+ {
+ if (!clsDict.HasKey("__module__"))
+ {
+ return null;
+ }
+ using var module = clsDict["__module__"];
+ return PyString.IsStringType(module) ? module.As() : null;
+ }
+
+ ///
+ /// The __file__ of the globals the class statement runs in, so same-named classes
+ /// from different scripts, which all run as __main__, never share a generated type.
+ ///
+ static string? DefinitionSite()
+ {
+ BorrowedReference globals = Runtime.PyEval_GetGlobals();
+ if (globals.IsNull)
+ {
+ return null;
+ }
+ BorrowedReference file = Runtime.PyDict_GetItemString(globals, "__file__");
+ return !file.IsNull && Runtime.PyString_Check(file) ? Runtime.GetManagedString(file) : null;
+ }
+
+ ///
+ /// Generates the managed type for a class that only implements .NET interfaces and
+ /// declares no __namespace__. Unlike an explicit request, a failure here must not
+ /// break the class statement, so it falls back to a plain Python type as before.
+ ///
+ ///
+ /// The type is shared only by classes of the same name and shape defined from the same
+ /// file, so re-running a script reuses it while another script's class never takes it over.
+ ///
+ static NewReference TryCreateInterfaceSubType(BorrowedReference name, ClassBase baseType, IList interfaces, PyDict clsDict)
+ {
+ try
+ {
+ NewReference derived = TypeManager.CreateSubType(name, baseType, interfaces, clsDict, ModuleNamespace(clsDict), DefinitionSite());
+ if (!derived.IsNull())
+ {
+ return derived;
+ }
+ }
+ catch (Exception)
+ {
+ }
+ Exceptions.Clear();
+ return default;
+ }
+
///
/// Metatype __new__ implementation. This is called to create a new
/// class / type when a reflected class is subclassed.
@@ -120,7 +171,8 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args,
}
}
// if the base type count is 0, there might still be interfaces to implement.
- if (baseTypes.Count == 0)
+ bool implementsInterfacesOnly = baseTypes.Count == 0;
+ if (implementsInterfacesOnly)
{
baseTypes.Add(new ClassBase(typeof(object)));
}
@@ -172,7 +224,15 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args,
using var clsDict = new PyDict(dict);
if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__"))
{
- return TypeManager.CreateSubType(name, baseTypes[0], interfaces, clsDict);
+ return RunClassCreationHooks(TypeManager.CreateSubType(name, baseTypes[0], interfaces, clsDict), dict, kw);
+ }
+ if (implementsInterfacesOnly)
+ {
+ NewReference derived = TryCreateInterfaceSubType(name, baseTypes[0], interfaces, clsDict);
+ if (!derived.IsNull())
+ {
+ return RunClassCreationHooks(derived, dict, kw);
+ }
}
}
@@ -221,6 +281,57 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args,
}
+ ///
+ /// Finishes a class whose type was generated rather than built by type.__new__, by
+ /// running the class-creation protocol type.__new__ would: __set_name__ on
+ /// every namespace value that defines it, then the parent's __init_subclass__ with
+ /// the class keywords.
+ ///
+ /// Takes ownership of ; returns null with the Python error set on failure.
+ static NewReference RunClassCreationHooks(NewReference type, BorrowedReference namespaceDict, BorrowedReference kw)
+ {
+ if (type.IsNull())
+ {
+ return type;
+ }
+ try
+ {
+ using var cls = new PyObject(type.Borrow());
+ using var namespaceItems = new PyDict(namespaceDict);
+ using (var members = namespaceItems.Items())
+ {
+ foreach (PyObject member in members)
+ {
+ using (member)
+ using (var memberName = member[0])
+ using (var value = member[1])
+ using (var valueType = value.GetPythonType())
+ {
+ if (!valueType.HasAttr("__set_name__"))
+ {
+ continue;
+ }
+ using var setName = valueType.GetAttr("__set_name__");
+ setName.Invoke(value, cls, memberName).Dispose();
+ }
+ }
+ }
+ using var builtins = PyModule.Import("builtins");
+ using var super = builtins.GetAttr("super");
+ using var parent = super.Invoke(cls, cls);
+ using var initSubclass = parent.GetAttr("__init_subclass__");
+ using var keywords = kw.IsNull ? null : new PyDict(kw);
+ initSubclass.Invoke(Array.Empty(), keywords).Dispose();
+ return type;
+ }
+ catch (PythonException error)
+ {
+ type.Dispose();
+ error.Restore();
+ return default;
+ }
+ }
+
public static NewReference tp_alloc(BorrowedReference mt, nint n)
=> Runtime.PyType_GenericAlloc(mt, n);
diff --git a/src/runtime/Types/PropertyObject.cs b/src/runtime/Types/PropertyObject.cs
index f09d1696a..3049c826a 100644
--- a/src/runtime/Types/PropertyObject.cs
+++ b/src/runtime/Types/PropertyObject.cs
@@ -124,7 +124,7 @@ public static int tp_descr_set(BorrowedReference ds, BorrowedReference ob, Borro
}
- if (!Converter.ToManaged(val, info.PropertyType, out var newval, true))
+ if (!Converter.ToManagedTruthy(val, info.PropertyType, out var newval, true))
{
return -1;
}
diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs
index 3d0aa7e99..3dede7bc1 100644
--- a/src/runtime/Types/ReflectedClrType.cs
+++ b/src/runtime/Types/ReflectedClrType.cs
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
+using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using static Python.Runtime.PythonException;
@@ -70,7 +71,7 @@ internal void Restore(ClassBase cb)
internal static NewReference CreateSubclass(ClassBase baseClass, IList interfaces,
string name, string? assembly, string? ns,
- BorrowedReference dict)
+ BorrowedReference dict, string? definitionSite = null)
{
try
{
@@ -79,8 +80,10 @@ internal static NewReference CreateSubclass(ClassBase baseClass, IList int
interfaces,
dict,
ns,
- assembly);
+ assembly,
+ definitionSite: definitionSite);
+ ClassManager.cache.Remove(subType);
var py_type = GetOrCreate(subType);
// by default the class dict will have all the C# methods in it, but as this is a
@@ -94,6 +97,10 @@ internal static NewReference CreateSubclass(ClassBase baseClass, IList int
ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type));
ThrowIfIsNotZero(Runtime.PyDict_DelItemString(cls_dict, "__classcell__"));
}
+ WrapAsClassMethod(cls_dict, "__init_subclass__");
+ AddInstanceDictDescriptor(py_type, cls_dict);
+ WrapAsClassMethod(cls_dict, "__class_getitem__");
+ Runtime.PyType_Modified(py_type);
return new NewReference(py_type);
}
@@ -103,6 +110,57 @@ internal static NewReference CreateSubclass(ClassBase baseClass, IList int
}
}
+ ///
+ /// Makes a plain function stored under a classmethod, as
+ /// type.__new__ does for __init_subclass__ and __class_getitem__.
+ ///
+ static void WrapAsClassMethod(BorrowedReference classDict, string name)
+ {
+ BorrowedReference member = Runtime.PyDict_GetItemString(classDict, name);
+ if (member.IsNull)
+ {
+ return;
+ }
+ using var types = PyModule.Import("types");
+ using var functionType = types.GetAttr("FunctionType");
+ if (Runtime.PyObject_TYPE(member) != functionType.Reference)
+ {
+ return;
+ }
+ using var builtins = PyModule.Import("builtins");
+ using var classMethod = builtins.GetAttr("classmethod");
+ using var function = new PyObject(member);
+ using var wrapped = classMethod.Invoke(function);
+ ThrowIfIsNotZero(Runtime.PyDict_SetItemString(classDict, name, wrapped));
+ }
+
+ static IntPtr instanceDictGetSet;
+
+ ///
+ /// Exposes the instance dictionary as __dict__, which type.__new__ would have
+ /// added, so vars() and descriptors such as functools.cached_property work.
+ ///
+ static void AddInstanceDictDescriptor(PyType type, BorrowedReference classDict)
+ {
+ if (Util.ReadInt32(type, TypeOffset.tp_dictoffset) <= 0
+ || !Runtime.PyDict_GetItemString(classDict, "__dict__").IsNull)
+ {
+ return;
+ }
+ if (instanceDictGetSet == IntPtr.Zero)
+ {
+ IntPtr getSet = Marshal.AllocHGlobal(5 * IntPtr.Size);
+ Marshal.WriteIntPtr(getSet, 0, Marshal.StringToHGlobalAnsi("__dict__"));
+ Marshal.WriteIntPtr(getSet, IntPtr.Size, Runtime.PyObject_GenericGetDictAddress);
+ Marshal.WriteIntPtr(getSet, 2 * IntPtr.Size, Runtime.PyObject_GenericSetDictAddress);
+ Marshal.WriteIntPtr(getSet, 3 * IntPtr.Size, IntPtr.Zero);
+ Marshal.WriteIntPtr(getSet, 4 * IntPtr.Size, IntPtr.Zero);
+ instanceDictGetSet = getSet;
+ }
+ using var descriptor = Runtime.PyDescr_NewGetSet(type, instanceDictGetSet);
+ ThrowIfIsNotZero(Runtime.PyDict_SetItemString(classDict, "__dict__", descriptor.BorrowOrThrow()));
+ }
+
static ReflectedClrType AllocateClass(Type clrType)
{
string name = TypeManager.GetPythonTypeName(clrType);
diff --git a/tests/test_subclass.py b/tests/test_subclass.py
index c6ab7650f..111ac7af2 100644
--- a/tests/test_subclass.py
+++ b/tests/test_subclass.py
@@ -29,6 +29,22 @@ def bar(self, x, i):
return InterfaceTestClass
+def redefine_interface_test_class_fixture(subnamespace):
+ """Redefine InterfaceTestClass with the same CLR identity and new behaviour."""
+
+ class InterfaceTestClass(IInterfaceTest):
+ """class that implements the test interface"""
+ __namespace__ = "Python.Test." + subnamespace
+
+ def foo(self):
+ return "InterfaceTestClass2"
+
+ def bar(self, x, i):
+ return "/".join([x] * i)
+
+ return InterfaceTestClass
+
+
def interface_generic_class_fixture(subnamespace):
class GenericInterfaceImpl(IGenericInterface[int]):
@@ -146,6 +162,20 @@ def test_derived_class():
x = FunctionsTest.pass_through(ob)
assert id(x) == id(ob)
+
+def test_redefine_class():
+ """Classes with the same CLR identity keep working after redefinition."""
+ InterfaceTestClass = interface_test_class_fixture(test_redefine_class.__name__)
+ ob = InterfaceTestClass()
+ assert ob.foo() == "InterfaceTestClass"
+
+ InterfaceTestClass = redefine_interface_test_class_fixture(
+ test_redefine_class.__name__
+ )
+ ob = InterfaceTestClass()
+ assert ob.foo() == "InterfaceTestClass2"
+
+
def test_broken_derived_class():
"""Test python class derived from managed type with invalid namespace"""
with pytest.raises(TypeError):