Runtime fixes for pyrevit.forms under CPython (pyRevit #3846)

src/runtime/Codecs/PyObjectConversions.cs

@@ -13,6 +13,10 @@ namespace Python.Runtime
     /// This class allows to register additional marshalling codecs.
     /// <para>Python.NET will pick suitable encoder/decoder registered first</para>
     /// </summary>
+    /// <remarks>
+    /// Codecs' <c>CanDecode</c> and <c>CanEncode</c> run outside the registry lock, because a
+    /// codec may run Python code, which can hand the GIL to a thread already waiting on that lock.
+    /// </remarks>
     public static class PyObjectConversions
     {
         static readonly DecoderGroup decoders = new();
@@ -29,6 +33,7 @@ public static void RegisterEncoder(IPyObjectEncoder encoder)
             lock (encoders)
             {
                 encoders.Add(encoder);
+                clrToPython.Clear();
             }
         }
 
@@ -43,6 +48,7 @@ public static void RegisterDecoder(IPyObjectDecoder decoder)
             lock (decoders)
             {
                 decoders.Add(decoder);
+                pythonToClr.Clear();
             }
         }
 
@@ -65,10 +71,12 @@ static readonly ConcurrentDictionary<Type, IPyObjectEncoder[]>
             clrToPython = new();
         static IPyObjectEncoder[] GetEncoders(Type type)
         {
+            IPyObjectEncoder[] registered;
             lock (encoders)
             {
-                return encoders.GetEncoders(type).ToArray();
+                registered = encoders.ToArray();
             }
+            return registered.SelectMany(encoder => encoder.GetEncoders(type)).ToArray();
         }
         #endregion
 
@@ -93,12 +101,15 @@ internal static bool TryDecode(BorrowedReference pyHandle, BorrowedReference pyT
             Debug.Assert(PyType.IsType(sourceTypeRef));
             var pyType = new PyType(sourceTypeRef, prevalidated: true);
 
-            IPyObjectDecoder? decoder;
+            IPyObjectDecoder[] registered;
             lock (decoders)
             {
-                decoder = decoders.GetDecoder(pyType, targetType);
-                if (decoder == null) return default;
+                registered = decoders.ToArray();
             }
+            IPyObjectDecoder? decoder = registered
+                .Select(candidate => candidate.GetDecoder(pyType, targetType))
+                .FirstOrDefault(found => found != null);
+            if (decoder == null) return default;
 
             var decode = genericDecode.MakeGenericMethod(targetType)!;
 

src/runtime/Converter.cs

@@ -65,6 +65,9 @@ static Converter()
             if (op == Runtime.PyBoolType)
                 return boolType;
 
+            if (op == Runtime.PyBaseObjectType)
+                return objectType;
+
             return null;
         }
 
@@ -273,6 +276,63 @@ internal static bool ToManaged(BorrowedReference value, Type type,
             return Converter.ToManagedValue(value, type, out result, setError);
         }
 
+        [ThreadStatic]
+        private static int truthyBooleanScopes;
+
+        /// <summary>
+        /// Like <see cref="ToManaged(BorrowedReference, Type, out object?, bool)"/>, but a
+        /// <see cref="bool"/> or <see cref="Nullable{Boolean}"/> target also accepts any Python
+        /// object and converts it by truthiness, as IronPython does.
+        /// </summary>
+        /// <remarks>
+        /// Used where the target type is fixed: property and field assignment and delegate
+        /// return values. Method arguments use it only after no overload matched strictly, so a
+        /// <see cref="bool"/> overload never wins over an overload that takes the value itself.
+        /// </remarks>
+        internal static bool ToManagedTruthy(BorrowedReference value, Type type,
+            out object? result, bool setError)
+        {
+            using (AllowTruthyBooleans())
+            {
+                return ToManaged(value, type, out result, setError);
+            }
+        }
+
+        /// <summary>
+        /// Lets <see cref="bool"/> targets accept any Python object by truthiness on this thread
+        /// until the returned scope is disposed.
+        /// </summary>
+        internal static TruthyBooleanScope AllowTruthyBooleans()
+        {
+            truthyBooleanScopes++;
+            return default;
+        }
+
+        internal readonly struct TruthyBooleanScope : IDisposable
+        {
+            public void Dispose() => truthyBooleanScopes--;
+        }
+
+        internal static bool TakesBoolean(IEnumerable<MethodBase> methods)
+        {
+            foreach (MethodBase method in methods)
+            {
+                foreach (ParameterInfo parameter in method.GetParameters())
+                {
+                    Type type = parameter.ParameterType;
+                    if (type.IsByRef)
+                    {
+                        type = type.GetElementType();
+                    }
+                    if (type == boolType || type == typeof(bool?))
+                    {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+
         internal static bool ToManagedValue(BorrowedReference value, Type obType,
             out object? result, bool setError)
         {
@@ -612,6 +672,16 @@ internal static bool ToPrimitive(BorrowedReference value, Type obType, out objec
                         result = false;
                         return true;
                     }
+                    if (truthyBooleanScopes > 0)
+                    {
+                        int truth = Runtime.PyObject_IsTrue(value);
+                        if (truth < 0)
+                        {
+                            goto convert_error;
+                        }
+                        result = truth != 0;
+                        return true;
+                    }
                     if (setError)
                     {
                         goto type_error;

src/runtime/DelegateManager.cs

@@ -295,7 +295,7 @@ protected Dispatcher(PyObject target, Type dtype)
                             return null;
                         }
                         BorrowedReference item0 = Runtime.PyTuple_GetItem(op, 0);
-                        if (!Converter.ToManaged(item0, rtype, out object? result0, true))
+                        if (!Converter.ToManagedTruthy(item0, rtype, out object? result0, true))
                         {
                             Exceptions.RaiseTypeError($"The Python function returned a tuple where element 0 was not {rtype} (the return type)");
                             throw PythonException.ThrowLastAsClrException();
@@ -331,7 +331,7 @@ protected Dispatcher(PyObject target, Type dtype)
                     return null;
                 }
 
-                if (!Converter.ToManaged(op, rtype, out object? result, true))
+                if (!Converter.ToManagedTruthy(op, rtype, out object? result, true))
                 {
                     throw PythonException.ThrowLastAsClrException();
                 }

src/runtime/MethodBinder.cs

@@ -363,7 +363,21 @@ public MismatchedMethod(Exception exception, MethodBase mb)
                 _methods = GetMethods();
             }
 
-            return Bind(inst, args, kwargDict, _methods, matchGenerics: true);
+            Binding? binding = Bind(inst, args, kwargDict, _methods, matchGenerics: true);
+            if (binding is null && Converter.TakesBoolean(_methods))
+            {
+                PythonException? strictError = PythonException.FetchCurrentOrNullRaw();
+                using (Converter.AllowTruthyBooleans())
+                {
+                    binding = Bind(inst, args, kwargDict, _methods, matchGenerics: true);
+                }
+                if (binding is null && strictError is not null)
+                {
+                    Exceptions.Clear();
+                    strictError.Restore();
+                }
+            }
+            return binding;
         }
 
         static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary<string, PyObject> kwargDict, MethodBase[] methods, bool matchGenerics)

src/runtime/Mixins/CollectionMixinsProvider.cs

@@ -63,6 +63,11 @@ public IEnumerable<PyType> GetBaseTypes(Type type, IList<PyType> existingBases)
                 newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin")));
             }
 
+            if (interfaces.Contains(typeof(IDisposable)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("DisposableMixin")));
+            }
+
             if (newBases.Count == existingBases.Count)
             {
                 return existingBases;

src/runtime/Mixins/collections.py

@@ -16,6 +16,18 @@ def close(self):
 class IterableMixin(col.Iterable):
     pass
 
+class DisposableMixin:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if hasattr(self, 'Dispose'):
+            self.Dispose()
+        else:
+            from System import IDisposable
+            IDisposable(self).Dispose()
+        return False
+
 class SizedMixin(col.Sized):
     def __len__(self): return self.Count
 

src/runtime/Runtime.Delegates.cs

@@ -227,6 +227,8 @@ static Delegates()
             _PyType_Lookup = (delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference, BorrowedReference>)GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll));
             PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference, NewReference>)GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll));
             PyObject_GenericGetDict = (delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr, NewReference>)GetFunctionByName(nameof(PyObject_GenericGetDict), GetUnmanagedDll(PythonDLL));
+            PyObject_GenericSetDict = (delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference, IntPtr, int>)GetFunctionByName(nameof(PyObject_GenericSetDict), GetUnmanagedDll(PythonDLL));
+            PyDescr_NewGetSet = (delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr, NewReference>)GetFunctionByName(nameof(PyDescr_NewGetSet), GetUnmanagedDll(PythonDLL));
             PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference, BorrowedReference, int>)GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll));
             PyObject_GC_Del = (delegate* unmanaged[Cdecl]<StolenReference, void>)GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll));
             try
@@ -534,6 +536,8 @@ static Delegates()
         internal static delegate* unmanaged[Cdecl]<uint, BorrowedReference, int> PyThreadState_SetAsyncExcLLP64 { get; }
         internal static delegate* unmanaged[Cdecl]<ulong, BorrowedReference, int> PyThreadState_SetAsyncExcLP64 { get; }
         internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr, NewReference> PyObject_GenericGetDict { get; }
+        internal static delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference, IntPtr, int> PyObject_GenericSetDict { get; }
+        internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr, NewReference> PyDescr_NewGetSet { get; }
         internal static delegate* unmanaged[Cdecl]<BorrowedReference, TypeSlotID, IntPtr> PyType_GetSlot { get; }
         internal static delegate* unmanaged[Cdecl]<in NativeTypeSpec, BorrowedReference, NewReference> PyType_FromSpecWithBases { get; }
         internal static delegate* unmanaged[Cdecl]<BorrowedReference, void> _Py_NewReference { get; }

src/runtime/Runtime.cs

@@ -1647,6 +1647,12 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe
         internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero);
         internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context);
 
+        internal static IntPtr PyObject_GenericGetDictAddress => (IntPtr)Delegates.PyObject_GenericGetDict;
+
+        internal static IntPtr PyObject_GenericSetDictAddress => (IntPtr)Delegates.PyObject_GenericSetDict;
+
+        internal static NewReference PyDescr_NewGetSet(BorrowedReference type, IntPtr getset) => Delegates.PyDescr_NewGetSet(type, getset);
+
         internal static void PyObject_GC_Del(StolenReference ob) => Delegates.PyObject_GC_Del(ob);
 
 

src/runtime/TypeManager.cs

@@ -374,7 +374,7 @@ static PyTuple GetBaseTypeTuple(Type clrType)
             return new PyTuple(bases);
         }
 
-        internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList<Type> interfaces, BorrowedReference dictRef)
+        internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList<Type> interfaces, BorrowedReference dictRef, string? defaultNamespace = null, string? definitionSite = null)
         {
             // Utility to create a subtype of a managed type with the ability for the
             // a python subtype able to override the managed implementation
@@ -388,7 +388,7 @@ internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase
             // the derived class can have class attributes __assembly__ and __module__ which
             // control the name of the assembly and module the new type is created in.
             object? assembly = null;
-            object? namespaceStr = null;
+            object? namespaceStr = defaultNamespace;
 
             using (var assemblyKey = new PyString("__assembly__"))
             {
@@ -418,7 +418,8 @@ internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase
             return ReflectedClrType.CreateSubclass(py_base_type, interfaces, name,
                 ns: (string?)namespaceStr,
                 assembly: (string?)assembly,
-                dict: dictRef);
+                dict: dictRef,
+                definitionSite: definitionSite);
         }
 
         internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc)