From fa4135aa82c09756aa5be7cf928d4607dc72be5f Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:54:49 +0000 Subject: [PATCH] Don't hold a lock while lazy Python members run Python code PyDecimalType and the converter's datetime/tzinfo constructors were Lazy in ExecutionAndPublication mode. Their factories run Python code with the GIL held, and that code can hand the GIL to another thread. A thread that then reached the same Lazy blocked on its lock while holding the GIL, so the factory could never finish: a deadlock. PublicationOnly lets that thread run the factory itself; the first published value wins. Closes #153 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + src/embed_tests/TestLazyPyMembers.cs | 100 +++++++++++++++++++++++++++ src/runtime/Converter.cs | 5 +- src/runtime/Runtime.cs | 2 +- 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 src/embed_tests/TestLazyPyMembers.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 416b47c53..f1ac49d51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ Instead, `PyIterable` does that. ### Fixed +- Fix deadlock when several threads holding the GIL first resolve the Python `Decimal`, `datetime` or `tzinfo` types at the same time - Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash - Fixed parameterless .NET constructor being silently called when a matching constructor overload is not found ([#238][i238]) - Fix incorrect dereference in params array handling diff --git a/src/embed_tests/TestLazyPyMembers.cs b/src/embed_tests/TestLazyPyMembers.cs new file mode 100644 index 000000000..b563679e1 --- /dev/null +++ b/src/embed_tests/TestLazyPyMembers.cs @@ -0,0 +1,100 @@ +using System; +using System.Threading; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestLazyPyMembers + { + public static class OverloadedMethods + { + public static string Call(string value) => "string"; + public static string Call(object value) => "object"; + } + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + /// + /// Resolving PyDecimalType imports _pydecimal, which runs Python code that can hand the GIL + /// to another thread. A thread that reaches PyDecimalType meanwhile must not block on it + /// while holding the GIL, or the importing thread can never finish. + /// + [Test] + public void ConcurrentFirstResolutionOfDecimalTypeDoesNotDeadlock() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import sys +import threading +from Python.EmbeddingTest import TestLazyPyMembers +OverloadedMethods = TestLazyPyMembers.OverloadedMethods + +importing = threading.Event() + +class SlowDecimalImport: + def find_spec(self, name, path, target=None): + if name == '_pydecimal': + importing.set() + # keep the import open, with the GIL released, until the other thread reaches PyDecimalType + threading.Event().wait(1) + return None + +finder = SlowDecimalImport() +sys.modules.pop('_pydecimal', None) +sys.meta_path.insert(0, finder) +"); + Assert.IsFalse(Runtime.Runtime.PyDecimalType.IsValueCreated); + + var results = new string[2]; + var errors = new Exception[2]; + Thread Start(int index, string code) => new Thread(() => + { + try + { + using (Py.GIL()) + { + results[index] = scope.Eval(code); + } + } + catch (Exception e) + { + errors[index] = e; + } + }); + var first = Start(0, "OverloadedMethods.Call([])"); + var second = Start(1, "importing.wait(10) and OverloadedMethods.Call([])"); + + // the fixture initializes the engine with the GIL held on this thread: + // release it so the worker threads can acquire it + var ts = PythonEngine.BeginAllowThreads(); + try + { + second.Start(); + first.Start(); + Assert.IsTrue(first.Join(TimeSpan.FromSeconds(30)), "first thread deadlocked"); + Assert.IsTrue(second.Join(TimeSpan.FromSeconds(30)), "second thread deadlocked"); + } + finally + { + PythonEngine.EndAllowThreads(ts); + } + scope.Exec("sys.meta_path.remove(finder)"); + + Assert.IsNull(errors[0]); + Assert.IsNull(errors[1]); + Assert.AreEqual("object", results[0]); + Assert.AreEqual("object", results[1]); + } + } +} diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 806b6cbdd..e3c210144 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Threading; using Python.Runtime.Native; using System.Linq; @@ -149,7 +150,7 @@ def __reduce_ex__(self, protocol): var result = Runtime.PyObject_GetAttrString(dateTimeSubclassMod, "datetime").MoveToPyObject(); PythonException.ThrowIfIsNull(result); return result; - }); + }, LazyThreadSafetyMode.PublicationOnly); timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod.Borrow(), "timedelta").MoveToPyObject(); PythonException.ThrowIfIsNull(timeSpanCtor); @@ -172,7 +173,7 @@ def dst (self, dt): var result = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT").MoveToPyObject(); PythonException.ThrowIfIsNull(result); return result; - }); + }, LazyThreadSafetyMode.PublicationOnly); pyTupleNoKind = Runtime.PyTuple_New(7).MoveToPyObject(); pyTupleKind = Runtime.PyTuple_New(8).MoveToPyObject(); diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6fe8595dd..50cd73bb6 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -244,7 +244,7 @@ private static void InitPyMembers() var op = PyObject_CallObject(decimalCtor.BorrowNullable(), BorrowedReference.Null).MoveToPyObject(); SetPyMemberTypeOf(out var result, op); return result; - }); + }, LazyThreadSafetyMode.PublicationOnly); _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); {