diff --git a/CHANGELOG.md b/CHANGELOG.md index 3599c619b..bce1ec557 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - BREAKING: Names of .NET types (e.g. `str(__class__)`) changed to better support generic types - BREAKING: overload resolution will no longer prefer basic types. Instead, first matching overload will be chosen. +- BREAKING: `Exec` and `Eval` from `PythonEngine` no longer accept raw pointers. - BREAKING: .NET collections and arrays are no longer automatically converted to Python collections. Instead, they implement standard Python collection interfaces from `collections.abc`. @@ -74,6 +75,7 @@ be of type `PyInt` instead of `System.Int32` due to possible loss of information Python `float` will continue to be converted to `System.Double`. - BREAKING: Python.NET will no longer implicitly convert types like `numpy.float64`, that implement `__float__` to `System.Single` and `System.Double`. An explicit conversion is required on Python or .NET side. +- BREAKING: `PyObject.GetHashCode` can fail. - BREAKING: Python.NET will no longer implicitly convert any Python object to `System.Boolean`. - BREAKING: `PyObject.GetAttr(name, default)` now only ignores `AttributeError` (previously ignored all exceptions). - BREAKING: `PyObject` no longer implements `IEnumerable`. diff --git a/Directory.Build.props b/Directory.Build.props index 0f89af489..2130c35f9 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -13,7 +13,7 @@ all runtime; build; native; contentfiles; analyzers - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/pythonnet.sln b/pythonnet.sln index 5cf1d1cce..3b509518f 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -54,6 +54,9 @@ Global Release|Any CPU = Release|Any CPU Release|x64 = Release|x64 Release|x86 = Release|x86 + TraceAlloc|Any CPU = TraceAlloc|Any CPU + TraceAlloc|x64 = TraceAlloc|x64 + TraceAlloc|x86 = TraceAlloc|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU @@ -68,6 +71,12 @@ Global {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|x64.Build.0 = Release|Any CPU {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|x86.ActiveCfg = Release|Any CPU {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|x86.Build.0 = Release|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.TraceAlloc|Any CPU.ActiveCfg = TraceAlloc|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.TraceAlloc|Any CPU.Build.0 = TraceAlloc|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.TraceAlloc|x64.ActiveCfg = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.TraceAlloc|x64.Build.0 = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.TraceAlloc|x86.ActiveCfg = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.TraceAlloc|x86.Build.0 = Debug|Any CPU {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|Any CPU.ActiveCfg = Debug|x64 {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|Any CPU.Build.0 = Debug|x64 {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|x64.ActiveCfg = Debug|x64 @@ -80,6 +89,12 @@ Global {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|x64.Build.0 = Release|x64 {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|x86.ActiveCfg = Release|x86 {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|x86.Build.0 = Release|x86 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.TraceAlloc|Any CPU.ActiveCfg = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.TraceAlloc|Any CPU.Build.0 = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.TraceAlloc|x64.ActiveCfg = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.TraceAlloc|x64.Build.0 = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.TraceAlloc|x86.ActiveCfg = Debug|x86 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.TraceAlloc|x86.Build.0 = Debug|x86 {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|Any CPU.Build.0 = Debug|Any CPU {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -92,6 +107,12 @@ Global {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|x64.Build.0 = Release|Any CPU {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|x86.ActiveCfg = Release|Any CPU {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|x86.Build.0 = Release|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.TraceAlloc|Any CPU.ActiveCfg = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.TraceAlloc|Any CPU.Build.0 = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.TraceAlloc|x64.ActiveCfg = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.TraceAlloc|x64.Build.0 = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.TraceAlloc|x86.ActiveCfg = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.TraceAlloc|x86.Build.0 = Debug|Any CPU {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|Any CPU.Build.0 = Debug|Any CPU {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -104,6 +125,12 @@ Global {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|x64.Build.0 = Release|Any CPU {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|x86.ActiveCfg = Release|Any CPU {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|x86.Build.0 = Release|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.TraceAlloc|Any CPU.ActiveCfg = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.TraceAlloc|Any CPU.Build.0 = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.TraceAlloc|x64.ActiveCfg = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.TraceAlloc|x64.Build.0 = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.TraceAlloc|x86.ActiveCfg = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.TraceAlloc|x86.Build.0 = Debug|Any CPU {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|Any CPU.ActiveCfg = Debug|x64 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|Any CPU.Build.0 = Debug|x64 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|x64.ActiveCfg = Debug|x64 @@ -116,6 +143,12 @@ Global {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x64.Build.0 = Release|x64 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.ActiveCfg = Release|x86 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.Build.0 = Release|x86 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.TraceAlloc|Any CPU.ActiveCfg = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.TraceAlloc|Any CPU.Build.0 = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.TraceAlloc|x64.ActiveCfg = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.TraceAlloc|x64.Build.0 = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.TraceAlloc|x86.ActiveCfg = Debug|x86 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.TraceAlloc|x86.Build.0 = Debug|x86 {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = Debug|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -128,6 +161,12 @@ Global {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.TraceAlloc|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.TraceAlloc|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.TraceAlloc|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.TraceAlloc|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.TraceAlloc|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.TraceAlloc|x86.Build.0 = Debug|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.Build.0 = Debug|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -140,6 +179,12 @@ Global {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|x64.Build.0 = Release|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|x86.ActiveCfg = Release|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|x86.Build.0 = Release|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.TraceAlloc|Any CPU.ActiveCfg = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.TraceAlloc|Any CPU.Build.0 = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.TraceAlloc|x64.ActiveCfg = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.TraceAlloc|x64.Build.0 = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.TraceAlloc|x86.ActiveCfg = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.TraceAlloc|x86.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/embed_tests/CallableObject.cs b/src/embed_tests/CallableObject.cs index ab732be15..8466f5ad8 100644 --- a/src/embed_tests/CallableObject.cs +++ b/src/embed_tests/CallableObject.cs @@ -14,7 +14,7 @@ public void SetUp() { PythonEngine.Initialize(); using var locals = new PyDict(); - PythonEngine.Exec(CallViaInheritance.BaseClassSource, locals: locals.Handle); + PythonEngine.Exec(CallViaInheritance.BaseClassSource, locals: locals); CustomBaseTypeProvider.BaseClass = new PyType(locals[CallViaInheritance.BaseClassName]); PythonEngine.InteropConfiguration.PythonBaseTypeProviders.Add(new CustomBaseTypeProvider()); } diff --git a/src/embed_tests/Inheritance.cs b/src/embed_tests/Inheritance.cs index 41a73493e..950c08548 100644 --- a/src/embed_tests/Inheritance.cs +++ b/src/embed_tests/Inheritance.cs @@ -13,8 +13,8 @@ public class Inheritance public void SetUp() { PythonEngine.Initialize(); - var locals = new PyDict(); - PythonEngine.Exec(InheritanceTestBaseClassWrapper.ClassSourceCode, locals: locals.Handle); + using var locals = new PyDict(); + PythonEngine.Exec(InheritanceTestBaseClassWrapper.ClassSourceCode, locals: locals); ExtraBaseTypeProvider.ExtraBase = new PyType(locals[InheritanceTestBaseClassWrapper.ClassName]); var baseTypeProviders = PythonEngine.InteropConfiguration.PythonBaseTypeProviders; baseTypeProviders.Add(new ExtraBaseTypeProvider()); diff --git a/src/embed_tests/TestNamedArguments.cs b/src/embed_tests/TestNamedArguments.cs index 31f2ea1d2..c86302038 100644 --- a/src/embed_tests/TestNamedArguments.cs +++ b/src/embed_tests/TestNamedArguments.cs @@ -55,7 +55,7 @@ def Test3(self, a1 = 1, a2 = 1, a3 = 1, a4 = 1): return a1 + a2 + a3 + a4 a = cmTest3() -", null, locals.Handle); +", null, locals); return locals.GetItem("a"); } diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs index 700e73ae3..fa5fa38c7 100644 --- a/src/embed_tests/TestPyObject.cs +++ b/src/embed_tests/TestPyObject.cs @@ -46,7 +46,7 @@ def add(self, x, y): return x + y a = MemberNamesTest() -", null, locals.Handle); +", null, locals); PyObject a = locals.GetItem("a"); diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs index c6228f1b9..d1c9aac28 100644 --- a/src/embed_tests/TestPyWith.cs +++ b/src/embed_tests/TestPyWith.cs @@ -37,7 +37,7 @@ def fail(self): return 5 / 0 a = CmTest() -", null, locals.Handle); +", null, locals); var a = locals.GetItem("a"); @@ -76,7 +76,7 @@ def fail(self): return 5 / 0 a = CmTest() -", null, locals.Handle); +", null, locals); var a = locals.GetItem("a"); Py.With(a, cmTest => diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 9bf12b0a2..77696fd96 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -55,10 +55,12 @@ public static void RefCountTest() Assert.AreEqual(1, Runtime.Runtime.Refcount32(p)); // XIncref/XDecref increase and decrease RefCount +#pragma warning disable CS0618 // Type or member is obsolete. We are testing corresponding members Runtime.Runtime.XIncref(p); Assert.AreEqual(2, Runtime.Runtime.Refcount32(p)); - Runtime.Runtime.XDecref(p); + Runtime.Runtime.XDecref(op.Steal()); Assert.AreEqual(1, Runtime.Runtime.Refcount32(p)); +#pragma warning restore CS0618 // Type or member is obsolete op.Dispose(); diff --git a/src/embed_tests/pyrunstring.cs b/src/embed_tests/pyrunstring.cs index 4a83afa9a..57c133c00 100644 --- a/src/embed_tests/pyrunstring.cs +++ b/src/embed_tests/pyrunstring.cs @@ -37,7 +37,7 @@ public void TestEval() locals.SetItem("sys", sys); locals.SetItem("a", new PyInt(10)); - object b = PythonEngine.Eval("sys.attr1 + a + 1", null, locals.Handle) + object b = PythonEngine.Eval("sys.attr1 + a + 1", null, locals) .AsManagedObject(typeof(int)); Assert.AreEqual(111, b); } @@ -51,7 +51,7 @@ public void TestExec() locals.SetItem("sys", sys); locals.SetItem("a", new PyInt(10)); - PythonEngine.Exec("c = sys.attr1 + a + 1", null, locals.Handle); + PythonEngine.Exec("c = sys.attr1 + a + 1", null, locals); object c = locals.GetItem("c").AsManagedObject(typeof(int)); Assert.AreEqual(111, c); } diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 2b4e0a94c..98c151bab 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -1,6 +1,8 @@ namespace Python.Runtime { using System; + using System.Diagnostics; + /// /// Represents a reference to a Python object, that is being lent, and /// can only be safely used until execution returns to the caller. @@ -11,6 +13,7 @@ readonly ref struct BorrowedReference public bool IsNull => this.pointer == IntPtr.Zero; /// Gets a raw pointer to the Python object + [DebuggerHidden] public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; /// Gets a raw pointer to the Python object diff --git a/src/runtime/DefaultBaseTypeProvider.cs b/src/runtime/DefaultBaseTypeProvider.cs index 9a96660d9..08702c80f 100644 --- a/src/runtime/DefaultBaseTypeProvider.cs +++ b/src/runtime/DefaultBaseTypeProvider.cs @@ -24,7 +24,7 @@ static BorrowedReference GetBaseType(Type type) return Exceptions.Exception; return type.BaseType is not null - ? ClassManager.GetClass(type.BaseType).ObjectReference + ? ClassManager.GetClass(type.BaseType) : Runtime.PyBaseObjectType; } diff --git a/src/runtime/EventHandlerCollection.cs b/src/runtime/EventHandlerCollection.cs new file mode 100644 index 000000000..551893799 --- /dev/null +++ b/src/runtime/EventHandlerCollection.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.Serialization; +using System.Security.Permissions; + +namespace Python.Runtime; + +[Serializable] +internal class EventHandlerCollection: Dictionary> +{ + readonly EventInfo info; + public EventHandlerCollection(EventInfo @event) + { + info = @event; + } + + /// + /// Register a new Python object event handler with the event. + /// + internal bool AddEventHandler(BorrowedReference target, PyObject handler) + { + object? obj = null; + if (target != null) + { + var co = (CLRObject)ManagedType.GetManagedObject(target)!; + obj = co.inst; + } + + // Create a true delegate instance of the appropriate type to + // wrap the Python handler. Note that wrapper delegate creation + // always succeeds, though calling the wrapper may fail. + Type type = info.EventHandlerType; + Delegate d = PythonEngine.DelegateManager.GetDelegate(type, handler); + + // Now register the handler in a mapping from instance to pairs + // of (handler hash, delegate) so we can lookup to remove later. + object key = obj ?? info.ReflectedType; + if (!TryGetValue(key, out var list)) + { + list = new List(); + this[key] = list; + } + list.Add(new Handler(Runtime.PyObject_Hash(handler), d)); + + // Note that AddEventHandler helper only works for public events, + // so we have to get the underlying add method explicitly. + object[] args = { d }; + MethodInfo mi = info.GetAddMethod(true); + mi.Invoke(obj, BindingFlags.Default, null, args, null); + + return true; + } + + + /// + /// Remove the given Python object event handler. + /// + internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference handler) + { + object? obj = null; + if (target != null) + { + var co = (CLRObject)ManagedType.GetManagedObject(target)!; + obj = co.inst; + } + + nint hash = Runtime.PyObject_Hash(handler); + if (hash == -1 && Exceptions.ErrorOccurred()) + { + return false; + } + + object key = obj ?? info.ReflectedType; + + if (!TryGetValue(key, out var list)) + { + Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); + return false; + } + + object?[] args = { null }; + MethodInfo mi = info.GetRemoveMethod(true); + + for (var i = 0; i < list.Count; i++) + { + var item = (Handler)list[i]; + if (item.hash != hash) + { + continue; + } + args[0] = item.del; + try + { + mi.Invoke(obj, BindingFlags.Default, null, args, null); + } + catch + { + continue; + } + list.RemoveAt(i); + return true; + } + + Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); + return false; + } + + #region Serializable + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + protected EventHandlerCollection(SerializationInfo info, StreamingContext context) + : base(info, context) + { + this.info = (EventInfo)info.GetValue("event", typeof(EventInfo)); + } + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + base.GetObjectData(info, context); + + info.AddValue("event", this.info); + } + #endregion +} diff --git a/src/runtime/ManagedTypes.cd b/src/runtime/ManagedTypes.cd index 385ae7117..9a3e3de16 100644 --- a/src/runtime/ManagedTypes.cd +++ b/src/runtime/ManagedTypes.cd @@ -3,7 +3,7 @@ - FAAAAgAIAAAEDAAAAAAAAEACIACJAAIAAAAAAAIAAAQ= + VAAAAgAIAAAEDAAAAAAAAEACIACLAAIAAAAAAAIAoBU= classbase.cs @@ -17,7 +17,7 @@ - AAAAAAAAABAAAAAAAAAAACAAIAAJAAAAIAAAAACAAAI= + AAQAgAAAABAQAAAAAAAARGAAMgAJAAAAJAACACSABAI= arrayobject.cs @@ -31,28 +31,28 @@ - AAAACAAAAAAABABAAAAACAAAABAJAEAAAAAAAAIAAAA= + AAAACAAAAAAAAABAAAAACAAAABAJAAAAAAAAAAJAAAA= constructorbinding.cs - EAAAAAAAAAAAAAAAAAACAAACBIAAAAJAAAAAAAAAAAA= + AAAAAAAAAAAAAAAAAAACAAACAIAAAAIAAAAAAAAAABA= clrobject.cs - AAAAAEAgIAQABAAAAABAAAAAIAIAAAAAAhAQAAAAKBA= + AAAAAEIgIAQABAAAAABAAAAAIAIAAAAgAhAQAAAAKBA= moduleobject.cs - AAAACAAAAAAABAAAAAAACAAAABAJAEAAAAAAAAIAEAA= + AAAACAAAAAAAAAAAAAAACAAAABAJAAAAAAAAAAJAEAA= constructorbinding.cs @@ -74,14 +74,14 @@ - AAAAAAAAAAAADAAAIAAAEABAAAAAAAACAAAAAAIAAAQ= + AAAAAAAAAAAACAAAIAAAEABAAAAAAAACAAAAAAJAAAA= eventbinding.cs - AAACAAAAAAAAAAAAAAIAAIAAAAAEAAAAQABAAAIBEAQ= + AAACAAAAAAAAAAAAAAIAAIAAAAAEAAAAQABAAAJBEAA= eventobject.cs @@ -95,7 +95,7 @@ - AAAAAAAAAAAAAAAAAAAAAAACAAAAAEEBAAAAAAABAAQ= + AAAAAAAAAAAABAAAAAAAAAgCAAAAAEEBAAAAAABAABQ= extensiontype.cs @@ -116,14 +116,14 @@ - UCBBgoBAIUgAAAEAACAAsAACAgAIABIAQYAAACIYIBA= + UCBBgIAAAUgAAAAAASAAMACCAAAIABIAAZAAAAIYABA= managedtype.cs - AQAAAAAICBAAAQBAAAABAAIAAgABAAABAAAAUBCAAAQ= + AQAAAAAICBAAAQBAAAABAAIAAgABAAABAAAAUBCAABQ= metatype.cs @@ -138,14 +138,14 @@ - EAAAAAAAAIAADABAIAAAAAAAAAgBAAAAUgAAAAIAAAQ= + AAAAAAAAAIAACABAIAAAAAAAAAABAAAAUgAAAAJAAAE= methodbinding.cs - FIADAAAAAAAIBAAAIAAIAAAIAAgFAAAAUAAgAAIAEAQ= + BIADAAAAAAAIAAAAIAAIAAAIAAAFAAAAUAAgAAJAFAA= methodobject.cs @@ -159,7 +159,7 @@ - ECCCCkAAAAAABAAAAAABAAACAAAIAIIAEAAAAAIACAQ= + ECCCCkAAgAAAAAAAAAABAAgCAABIAAIBEAAAAAJACAA= moduleobject.cs @@ -188,7 +188,7 @@ - AAAAAAAAAAAAAAAAIAAAAAAAAAABAAAAAgAAAAIAAAQ= + AAAAAAAAAAAAAAAAIAAAAAAAAAABAAAAAgAAAAJAAAA= overload.cs diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index 16390ffaa..bbd021ad3 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -1,6 +1,7 @@ namespace Python.Runtime { using System; + using System.Diagnostics; using System.Diagnostics.Contracts; using System.Runtime.CompilerServices; @@ -13,6 +14,7 @@ ref struct NewReference IntPtr pointer; /// Creates a pointing to the same object + [DebuggerHidden] public NewReference(BorrowedReference reference, bool canBeNull = false) { var address = canBeNull @@ -68,6 +70,7 @@ public IntPtr DangerousMoveToPointerOrNull() /// that steals reference passed to it. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] + [DebuggerHidden] public StolenReference StealNullable() => StolenReference.TakeNullable(ref this.pointer); /// @@ -75,6 +78,7 @@ public IntPtr DangerousMoveToPointerOrNull() /// that steals reference passed to it. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] + [DebuggerHidden] public StolenReference Steal() { if (this.IsNull()) throw new NullReferenceException(); @@ -83,6 +87,7 @@ public StolenReference Steal() } [MethodImpl(MethodImplOptions.AggressiveInlining)] + [DebuggerHidden] public StolenReference StealOrThrow() { if (this.IsNull()) throw PythonException.ThrowLastAsClrException(); @@ -116,6 +121,7 @@ public static NewReference DangerousFromPointer(IntPtr pointer) internal static IntPtr DangerousGetAddress(in NewReference reference) => IsNull(reference) ? throw new NullReferenceException() : reference.pointer; [Pure] + [DebuggerHidden] internal static bool IsNull(in NewReference reference) => reference.pointer == IntPtr.Zero; } @@ -129,25 +135,26 @@ static class NewReferenceExtensions { /// Gets a raw pointer to the Python object [Pure] + [DebuggerHidden] public static IntPtr DangerousGetAddress(this in NewReference reference) => NewReference.DangerousGetAddress(reference); [Pure] + [DebuggerHidden] public static bool IsNull(this in NewReference reference) => NewReference.IsNull(reference); [Pure] + [DebuggerHidden] public static BorrowedReference BorrowNullable(this in NewReference reference) => new(NewReference.DangerousGetAddressOrNull(reference)); [Pure] + [DebuggerHidden] public static BorrowedReference Borrow(this in NewReference reference) => reference.IsNull() ? throw new NullReferenceException() : reference.BorrowNullable(); [Pure] + [DebuggerHidden] public static BorrowedReference BorrowOrThrow(this in NewReference reference) => reference.IsNull() ? throw PythonException.ThrowLastAsClrException() : reference.BorrowNullable(); - - [Obsolete] - public static NewReference AnalyzerWorkaround(this in NewReference reference) - => NewReference.DangerousFromPointer(reference.DangerousGetAddress()); } } diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index f7011ceb8..c90ca38e4 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -30,6 +30,12 @@ True true + + Debug;Release;TraceAlloc + + + + $(DefineConstants);TRACE_ALLOC diff --git a/src/runtime/ReflectedClrType.cs b/src/runtime/ReflectedClrType.cs new file mode 100644 index 000000000..3b83fb443 --- /dev/null +++ b/src/runtime/ReflectedClrType.cs @@ -0,0 +1,118 @@ +using System; +using System.Diagnostics; +using System.Runtime.Serialization; + +using static Python.Runtime.PythonException; + +namespace Python.Runtime; + +[Serializable] +internal sealed class ReflectedClrType : PyType +{ + private ReflectedClrType(StolenReference reference) : base(reference, prevalidated: true) { } + internal ReflectedClrType(ReflectedClrType original) : base(original, prevalidated: true) { } + ReflectedClrType(SerializationInfo info, StreamingContext context) : base(info, context) { } + + internal ClassBase Impl => (ClassBase)ManagedType.GetManagedObject(this)!; + + /// + /// Get the Python type that reflects the given CLR type. + /// + /// + /// Returned might be partially initialized. + /// If you need fully initialized type, use + /// + public static ReflectedClrType GetOrCreate(Type type) + { + if (ClassManager.cache.TryGetValue(type, out var pyType)) + { + return pyType; + } + + // Ensure, that matching Python type exists first. + // It is required for self-referential classes + // (e.g. with members, that refer to the same class) + pyType = AllocateClass(type); + ClassManager.cache.Add(type, pyType); + + var impl = ClassManager.CreateClass(type); + + TypeManager.InitializeClassCore(type, pyType, impl); + + ClassManager.InitClassBase(type, impl, pyType); + + // Now we force initialize the Python type object to reflect the given + // managed type, filling the Python type slots with thunks that + // point to the managed methods providing the implementation. + TypeManager.InitializeClass(pyType, impl, type); + + return pyType; + } + + internal void Restore(InterDomainContext context) + { + var cb = context.Storage.GetValue("impl"); + + cb.Load(this, context); + + Restore(cb); + } + + internal void Restore(ClassBase cb) + { + ClassManager.InitClassBase(cb.type.Value, cb, this); + + TypeManager.InitializeClass(this, cb, cb.type.Value); + } + + internal static NewReference CreateSubclass(ClassBase baseClass, + string name, string? assembly, string? ns, + BorrowedReference dict) + { + try + { + Type subType = ClassDerivedObject.CreateDerivedType(name, + baseClass.type.Value, + dict, + ns, + assembly); + + var py_type = GetOrCreate(subType); + + // by default the class dict will have all the C# methods in it, but as this is a + // derived class we want the python overrides in there instead if they exist. + var cls_dict = Util.ReadRef(py_type, TypeOffset.tp_dict); + ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, dict)); + // Update the __classcell__ if it exists + BorrowedReference cell = Runtime.PyDict_GetItemString(cls_dict, "__classcell__"); + if (!cell.IsNull) + { + ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type)); + ThrowIfIsNotZero(Runtime.PyDict_DelItemString(cls_dict, "__classcell__")); + } + + return new NewReference(py_type); + } + catch (Exception e) + { + return Exceptions.RaiseTypeError(e.Message); + } + } + + static ReflectedClrType AllocateClass(Type clrType) + { + string name = TypeManager.GetPythonTypeName(clrType); + + var type = TypeManager.AllocateTypeObject(name, Runtime.PyCLRMetaType); + type.Flags = TypeFlags.Default + | TypeFlags.HasClrInstance + | TypeFlags.HeapType + | TypeFlags.BaseType + | TypeFlags.HaveGC; + + return new ReflectedClrType(type.Steal()); + } + + public override bool Equals(PyObject? other) => other != null && rawPtr == other.rawPtr; + public override int GetHashCode() => rawPtr.GetHashCode(); +} diff --git a/src/runtime/StateSerialization/CLRMappedItem.cs b/src/runtime/StateSerialization/CLRMappedItem.cs new file mode 100644 index 000000000..ec050b119 --- /dev/null +++ b/src/runtime/StateSerialization/CLRMappedItem.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; + +namespace Python.Runtime; + +public class CLRMappedItem +{ + public object Instance { get; private set; } + public List PyRefs { get; set; } = new List(); + public bool Stored { get; set; } + + public CLRMappedItem(object instance) + { + Instance = instance; + } + + internal void AddRef(PyObject pyRef) + { + this.PyRefs.Add(pyRef); + } +} diff --git a/src/runtime/StateSerialization/CLRWrapperCollection.cs b/src/runtime/StateSerialization/CLRWrapperCollection.cs new file mode 100644 index 000000000..66d5170dd --- /dev/null +++ b/src/runtime/StateSerialization/CLRWrapperCollection.cs @@ -0,0 +1,21 @@ +using System.Collections.ObjectModel; + +namespace Python.Runtime; + +public class CLRWrapperCollection : KeyedCollection +{ + public bool TryGetValue(object key, out CLRMappedItem value) + { + if (Dictionary == null) + { + value = null; + return false; + } + return Dictionary.TryGetValue(key, out value); + } + + protected override object GetKeyForItem(CLRMappedItem item) + { + return item.Instance; + } +} diff --git a/src/runtime/StateSerialization/ClassManagerState.cs b/src/runtime/StateSerialization/ClassManagerState.cs index e278f658c..70bb076cd 100644 --- a/src/runtime/StateSerialization/ClassManagerState.cs +++ b/src/runtime/StateSerialization/ClassManagerState.cs @@ -6,6 +6,6 @@ namespace Python.Runtime.StateSerialization; [Serializable] internal class ClassManagerState { - public Dictionary Contexts { get; set; } - public Dictionary Cache { get; set; } + public Dictionary Contexts { get; set; } + public Dictionary Cache { get; set; } } diff --git a/src/runtime/StateSerialization/ICLRObjectStorer.cs b/src/runtime/StateSerialization/ICLRObjectStorer.cs new file mode 100644 index 000000000..b87339cd5 --- /dev/null +++ b/src/runtime/StateSerialization/ICLRObjectStorer.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace Python.Runtime; + +public interface ICLRObjectStorer +{ + ICollection Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage); + CLRWrapperCollection Restore(RuntimeDataStorage storage); +} diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index e14e74bbc..0a3fbef69 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -1,8 +1,6 @@ using System; using System.Reflection; using System.Runtime.Serialization; -using System.Runtime.Serialization.Formatters.Binary; -using System.IO; namespace Python.Runtime { @@ -12,21 +10,20 @@ internal struct MaybeMemberInfo : ISerializable where T : MemberInfo public static implicit operator MaybeMemberInfo(T ob) => new MaybeMemberInfo(ob); // .ToString() of the serialized object - const string SerializationName = "s"; + const string SerializationDescription = "d"; // The ReflectedType of the object const string SerializationType = "t"; - const string SerializationFieldName = "f"; - string name; - MemberInfo info; + const string SerializationMemberName = "n"; + MemberInfo? info; [NonSerialized] - Exception deserializationException; + Exception? deserializationException; public string DeletedMessage { get { - return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; + return $"The .NET {typeof(T).Name} {Description} no longer exists. Cause: " + deserializationException?.Message ; } } @@ -42,25 +39,26 @@ public T Value } } - public string Name => name; + public string Description { get; } public bool Valid => info != null; public override string ToString() { - return (info != null ? info.ToString() : $"missing type: {name}"); + return (info != null ? info.ToString() : $"missing: {Description}"); } public MaybeMemberInfo(T fi) { info = fi; - name = info?.ToString(); + Description = info?.ToString(); + if (info?.DeclaringType is not null) + Description += " of " + info.DeclaringType; deserializationException = null; } internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) { - // Assumption: name is always stored in "s" - name = serializationInfo.GetString(SerializationName); + Description = serializationInfo.GetString(SerializationDescription); info = null; deserializationException = null; try @@ -68,8 +66,8 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c var tp = Type.GetType(serializationInfo.GetString(SerializationType)); if (tp != null) { - var field_name = serializationInfo.GetString(SerializationFieldName); - MemberInfo mi = tp.GetField(field_name, ClassManager.BindingFlags); + var memberName = serializationInfo.GetString(SerializationMemberName); + MemberInfo? mi = Get(tp, memberName, ClassManager.BindingFlags); if (mi != null && ShouldBindMember(mi)) { info = mi; @@ -82,6 +80,15 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c } } + static MemberInfo? Get(Type type, string name, BindingFlags flags) + { + if (typeof(T) == typeof(FieldInfo)) + return type.GetField(name, flags); + if (typeof(T) == typeof(PropertyInfo)) + return type.GetProperty(name, flags); + throw new NotImplementedException(typeof(T).Name); + } + // This is complicated because we bind fields // based on the visibility of the field, properties // based on it's setter/getter (which is a method @@ -107,10 +114,10 @@ static bool ShouldBindMember(MemberInfo mi) public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - serializationInfo.AddValue(SerializationName, name); - if (Valid) + serializationInfo.AddValue(SerializationDescription, Description); + if (info is not null) { - serializationInfo.AddValue(SerializationFieldName, info.Name); + serializationInfo.AddValue(SerializationMemberName, info.Name); serializationInfo.AddValue(SerializationType, info.ReflectedType.AssemblyQualifiedName); } } diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 1f7e94033..e773b8e03 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -68,6 +68,9 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c name = serializationInfo.GetString(SerializationName); info = null; deserializationException = null; + + if (name is null) return; + try { // Retrieve the reflected type of the method; diff --git a/src/runtime/StateSerialization/SharedObjectsState.cs b/src/runtime/StateSerialization/SharedObjectsState.cs index 2c79f5dfa..a445c9252 100644 --- a/src/runtime/StateSerialization/SharedObjectsState.cs +++ b/src/runtime/StateSerialization/SharedObjectsState.cs @@ -6,8 +6,8 @@ namespace Python.Runtime.StateSerialization; [Serializable] internal class SharedObjectsState { - public List InternalStores { get; set; } - public List Extensions { get; set; } + public Dictionary InternalStores { get; set; } + public Dictionary Extensions { get; set; } public RuntimeDataStorage Wrappers { get; set; } public Dictionary Contexts { get; set; } } diff --git a/src/runtime/StateSerialization/TypeManagerState.cs b/src/runtime/StateSerialization/TypeManagerState.cs index 9faf4e2f7..158579549 100644 --- a/src/runtime/StateSerialization/TypeManagerState.cs +++ b/src/runtime/StateSerialization/TypeManagerState.cs @@ -7,5 +7,4 @@ namespace Python.Runtime.StateSerialization; internal class TypeManagerState { public Dictionary Cache { get; set; } - public Dictionary SlotImplementations { get; set; } } diff --git a/src/runtime/StolenReference.cs b/src/runtime/StolenReference.cs index 39326bcfd..49304c1fd 100644 --- a/src/runtime/StolenReference.cs +++ b/src/runtime/StolenReference.cs @@ -1,6 +1,7 @@ namespace Python.Runtime { using System; + using System.Diagnostics; using System.Diagnostics.Contracts; using System.Runtime.CompilerServices; @@ -13,6 +14,7 @@ readonly ref struct StolenReference { internal readonly IntPtr Pointer; + [DebuggerHidden] StolenReference(IntPtr pointer) { Pointer = pointer; @@ -25,6 +27,7 @@ public static StolenReference Take(ref IntPtr ptr) return TakeNullable(ref ptr); } [MethodImpl(MethodImplOptions.AggressiveInlining)] + [DebuggerHidden] public static StolenReference TakeNullable(ref IntPtr ptr) { var stolenAddr = ptr; @@ -62,12 +65,14 @@ public static StolenReference DangerousFromPointer(IntPtr ptr) static class StolenReferenceExtensions { [Pure] + [DebuggerHidden] public static IntPtr DangerousGetAddressOrNull(this in StolenReference reference) => reference.Pointer; [Pure] + [DebuggerHidden] public static IntPtr DangerousGetAddress(this in StolenReference reference) => reference.Pointer == IntPtr.Zero ? throw new NullReferenceException() : reference.Pointer; - + [DebuggerHidden] public static StolenReference AnalyzerWorkaround(this in StolenReference reference) { IntPtr ptr = reference.DangerousGetAddressOrNull(); diff --git a/src/runtime/UnloadedClass.cs b/src/runtime/UnloadedClass.cs new file mode 100644 index 000000000..858045304 --- /dev/null +++ b/src/runtime/UnloadedClass.cs @@ -0,0 +1,27 @@ +using System; + +namespace Python.Runtime; + +[Serializable] +internal class UnloadedClass : ClassBase +{ + readonly string name; + + internal UnloadedClass(string name) : base(typeof(object)) + { + this.name = name; + } + + public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, BorrowedReference kw) + { + var self = (UnloadedClass)GetManagedObject(tp)!; + return self.RaiseTypeError(); + } + + public override NewReference type_subscript(BorrowedReference idx) => RaiseTypeError(); + + private NewReference RaiseTypeError() + => Exceptions.RaiseTypeError("The .NET type no longer exists: " + name); + + internal override bool CanSubclass() => false; +} diff --git a/src/runtime/UnsafeReferenceWithRun.cs b/src/runtime/UnsafeReferenceWithRun.cs new file mode 100644 index 000000000..665f4a3c6 --- /dev/null +++ b/src/runtime/UnsafeReferenceWithRun.cs @@ -0,0 +1,27 @@ +using System; +using System.ComponentModel; + +namespace Python.Runtime; + +[EditorBrowsable(EditorBrowsableState.Never)] +[Obsolete(Util.InternalUseOnly)] +public struct UnsafeReferenceWithRun +{ + internal UnsafeReferenceWithRun(BorrowedReference pyObj) + { + RawObj = pyObj.DangerousGetAddressOrNull(); + Run = Runtime.GetRun(); + } + + internal IntPtr RawObj; + internal BorrowedReference Ref => new(RawObj); + internal int Run; + + internal BorrowedReference CheckRun() + { + if (Run != Runtime.GetRun()) + throw new RuntimeShutdownException(RawObj); + + return Ref; + } +} diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index d0407e550..6fd467ae7 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -28,11 +28,13 @@ internal static class Util [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int ReadInt32(BorrowedReference ob, int offset) { + Debug.Assert(offset >= 0); return Marshal.ReadInt32(ob.DangerousGetAddress(), offset); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static long ReadInt64(BorrowedReference ob, int offset) { + Debug.Assert(offset >= 0); return Marshal.ReadInt64(ob.DangerousGetAddress(), offset); } @@ -40,6 +42,7 @@ internal static long ReadInt64(BorrowedReference ob, int offset) internal unsafe static T* ReadPtr(BorrowedReference ob, int offset) where T: unmanaged { + Debug.Assert(offset >= 0); IntPtr ptr = Marshal.ReadIntPtr(ob.DangerousGetAddress(), offset); return (T*)ptr; } @@ -47,39 +50,46 @@ internal static long ReadInt64(BorrowedReference ob, int offset) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe static IntPtr ReadIntPtr(BorrowedReference ob, int offset) { + Debug.Assert(offset >= 0); return Marshal.ReadIntPtr(ob.DangerousGetAddress(), offset); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe static BorrowedReference ReadRef(BorrowedReference @ref, int offset) { + Debug.Assert(offset >= 0); return new BorrowedReference(ReadIntPtr(@ref, offset)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void WriteInt32(BorrowedReference ob, int offset, int value) { + Debug.Assert(offset >= 0); Marshal.WriteInt32(ob.DangerousGetAddress(), offset, value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void WriteInt64(BorrowedReference ob, int offset, long value) { + Debug.Assert(offset >= 0); Marshal.WriteInt64(ob.DangerousGetAddress(), offset, value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe static void WriteIntPtr(BorrowedReference ob, int offset, IntPtr value) { + Debug.Assert(offset >= 0); Marshal.WriteIntPtr(ob.DangerousGetAddress(), offset, value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe static void WriteRef(BorrowedReference ob, int offset, in StolenReference @ref) { + Debug.Assert(offset >= 0); Marshal.WriteIntPtr(ob.DangerousGetAddress(), offset, @ref.DangerousGetAddress()); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe static void WriteNullableRef(BorrowedReference ob, int offset, in StolenReference @ref) { + Debug.Assert(offset >= 0); Marshal.WriteIntPtr(ob.DangerousGetAddress(), offset, @ref.DangerousGetAddressOrNull()); } diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 8bde70401..3ca09ddce 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -11,7 +11,7 @@ namespace Python.Runtime /// to support natural array usage (indexing) from Python. /// [Serializable] - internal class ArrayObject : ClassBase + internal sealed class ArrayObject : ClassBase { internal ArrayObject(Type tp) : base(tp) { @@ -99,6 +99,15 @@ static NewReference CreateMultidimensional(Type elementType, long[] dimensions, static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, long[] dimensions) { + for (int dim = 0; dim < dimensions.Length; dim++) + { + if (dimensions[dim] < 0) + { + Exceptions.SetError(Exceptions.ValueError, $"Non-negative number required (dims[{dim}])"); + return default; + } + } + object result; try { @@ -131,10 +140,10 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, /// /// Implements __getitem__ for array types. /// - public new static NewReference mp_subscript(BorrowedReference ob, BorrowedReference idx) + public static NewReference mp_subscript(BorrowedReference ob, BorrowedReference idx) { var obj = (CLRObject)GetManagedObject(ob)!; - var arrObj = (ArrayObject)GetManagedObjectType(ob)!; + var arrObj = (ArrayObject)GetManagedObject(Runtime.PyObject_TYPE(ob))!; if (!arrObj.type.Valid) { return Exceptions.RaiseTypeError(arrObj.type.DeletedMessage); @@ -142,7 +151,7 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, var items = (Array)obj.inst; Type itemType = arrObj.type.Value.GetElementType(); int rank = items.Rank; - nint index; + long index; object value; // Note that CLR 1.0 only supports int indexes - methods to @@ -169,19 +178,17 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, if (index < 0) { - index = items.Length + index; + index = items.LongLength + index; } - try - { - value = items.GetValue(index); - } - catch (IndexOutOfRangeException) + if (index < 0 || index >= items.LongLength) { Exceptions.SetError(Exceptions.IndexError, "array index out of range"); return default; } + value = items.GetValue(index); + return Converter.ToPython(value, itemType); } @@ -211,23 +218,23 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, return Exceptions.RaiseTypeError("invalid index value"); } + long len = items.GetLongLength(dimension); + if (index < 0) { - index = items.GetLength(dimension) + index; + index = len + index; + } + + if (index < 0 || index >= len) + { + Exceptions.SetError(Exceptions.IndexError, "array index out of range"); + return default; } indices[dimension] = index; } - try - { - value = items.GetValue(indices); - } - catch (IndexOutOfRangeException) - { - Exceptions.SetError(Exceptions.IndexError, "array index out of range"); - return default; - } + value = items.GetValue(indices); return Converter.ToPython(value, itemType); } @@ -236,13 +243,13 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, /// /// Implements __setitem__ for array types. /// - public static new int mp_ass_subscript(BorrowedReference ob, BorrowedReference idx, BorrowedReference v) + public static int mp_ass_subscript(BorrowedReference ob, BorrowedReference idx, BorrowedReference v) { var obj = (CLRObject)GetManagedObject(ob)!; var items = (Array)obj.inst; Type itemType = obj.inst.GetType().GetElementType(); int rank = items.Rank; - nint index; + long index; object? value; if (items.IsReadOnly) @@ -273,19 +280,16 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, if (index < 0) { - index = items.Length + index; + index = items.LongLength + index; } - try - { - items.SetValue(value, index); - } - catch (IndexOutOfRangeException) + if (index < 0 || index >= items.LongLength) { Exceptions.SetError(Exceptions.IndexError, "array index out of range"); return -1; } + items.SetValue(value, index); return 0; } @@ -314,23 +318,23 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, return -1; } + long len = items.GetLongLength(dimension); + if (index < 0) { - index = items.GetLength(dimension) + index; + index = len + index; + } + + if (index < 0 || index >= len) + { + Exceptions.SetError(Exceptions.IndexError, "array index out of range"); + return -1; } indices[dimension] = index; } - try - { - items.SetValue(value, indices); - } - catch (IndexOutOfRangeException) - { - Exceptions.SetError(Exceptions.IndexError, "array index out of range"); - return -1; - } + items.SetValue(value, indices); return 0; } diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index d9f332346..069757b40 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -4,6 +4,10 @@ using System.Diagnostics; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; + +using Python.Runtime.Slots; namespace Python.Runtime { @@ -16,20 +20,14 @@ namespace Python.Runtime /// each variety of reflected type. /// [Serializable] - internal class ClassBase : ManagedType + internal class ClassBase : ManagedType, IDeserializationCallback { [NonSerialized] - internal readonly List dotNetMembers = new(); + internal List dotNetMembers = new(); internal Indexer? indexer; internal readonly Dictionary richcompare = new(); internal MaybeType type; - internal new PyType pyHandle - { - get => (PyType)base.pyHandle; - set => base.pyHandle = value; - } - internal ClassBase(Type tp) { if (tp is null) throw new ArgumentNullException(nameof(type)); @@ -83,8 +81,8 @@ public virtual NewReference type_subscript(BorrowedReference idx) { return Exceptions.RaiseTypeError(e.Message); } - ManagedType c = ClassManager.GetClass(t); - return new NewReference(c.ObjectReference); + var c = ClassManager.GetClass(t); + return new NewReference(c); } return Exceptions.RaiseTypeError($"{type.Value.Namespace}.{type.Name} does not accept {types.Length} generic parameters"); @@ -114,8 +112,8 @@ public static NewReference tp_richcompare(BorrowedReference ob, BorrowedReferenc { case Runtime.Py_EQ: case Runtime.Py_NE: - PyObject pytrue = Runtime.PyTrue; - PyObject pyfalse = Runtime.PyFalse; + BorrowedReference pytrue = Runtime.PyTrue; + BorrowedReference pyfalse = Runtime.PyFalse; // swap true and false for NE if (op != Runtime.Py_EQ) @@ -165,7 +163,7 @@ public static NewReference tp_richcompare(BorrowedReference ob, BorrowedReferenc { int cmp = co1Comp.CompareTo(co2.inst); - PyObject pyCmp; + BorrowedReference pyCmp; if (cmp < 0) { if (op == Runtime.Py_LT || op == Runtime.Py_LE) @@ -215,7 +213,7 @@ public static NewReference tp_richcompare(BorrowedReference ob, BorrowedReferenc /// allows natural iteration over objects that either are IEnumerable /// or themselves support IEnumerator directly. /// - public static NewReference tp_iter(BorrowedReference ob) + static NewReference tp_iter_impl(BorrowedReference ob) { var co = GetManagedObject(ob) as CLRObject; if (co == null) @@ -254,7 +252,7 @@ public static NewReference tp_iter(BorrowedReference ob) } } - return new NewReference(new Iterator(o, elemType).ObjectReference); + return new Iterator(o, elemType).Alloc(); } @@ -339,54 +337,32 @@ public static NewReference tp_repr(BorrowedReference ob) /// public static void tp_dealloc(NewReference lastRef) { - ManagedType self = GetManagedObject(lastRef.Borrow())!; - tp_clear(lastRef.Borrow()); Runtime.PyObject_GC_UnTrack(lastRef.Borrow()); - Runtime.PyObject_GC_Del(lastRef.Steal()); - self?.FreeGCHandle(); - } - public static int tp_clear(BorrowedReference ob) - { - if (GetManagedObject(ob) is { } self) - { - if (self.clearReentryGuard) return 0; + CallClear(lastRef.Borrow()); - // workaround for https://bugs.python.org/issue45266 - self.clearReentryGuard = true; + IntPtr addr = lastRef.DangerousGetAddress(); + bool deleted = CLRObject.reflectedObjects.Remove(addr); + Debug.Assert(deleted); - try - { - return ClearImpl(ob, self); - } - finally - { - self.clearReentryGuard = false; - } - } - else - { - return ClearImpl(ob, null); - } + DecrefTypeAndFree(lastRef.Steal()); } - static int ClearImpl(BorrowedReference ob, ManagedType? self) + public static int tp_clear(BorrowedReference ob) { - bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType; - if (!isTypeObject) - { - int baseClearResult = BaseUnmanagedClear(ob); - if (baseClearResult != 0) - { - return baseClearResult; - } + TryFreeGCHandle(ob); - ClearObjectDict(ob); + int baseClearResult = BaseUnmanagedClear(ob); + if (baseClearResult != 0) + { + return baseClearResult; } + + ClearObjectDict(ob); return 0; } - static unsafe int BaseUnmanagedClear(BorrowedReference ob) + internal static unsafe int BaseUnmanagedClear(BorrowedReference ob) { var type = Runtime.PyObject_TYPE(ob); var unmanagedBase = GetUnmanagedBaseType(type); @@ -396,36 +372,42 @@ static unsafe int BaseUnmanagedClear(BorrowedReference ob) return 0; } var clear = (delegate* unmanaged[Cdecl])clearPtr; + + bool usesSubtypeClear = clearPtr == TypeManager.subtype_clear; + if (usesSubtypeClear) + { + // workaround for https://bugs.python.org/issue45266 (subtype_clear) + using var dict = Runtime.PyObject_GenericGetDict(ob); + if (Runtime.PyMapping_HasKey(dict.Borrow(), PyIdentifier.__clear_reentry_guard__) != 0) + return 0; + int res = Runtime.PyDict_SetItem(dict.Borrow(), PyIdentifier.__clear_reentry_guard__, Runtime.None); + if (res != 0) return res; + + res = clear(ob); + Runtime.PyDict_DelItem(dict.Borrow(), PyIdentifier.__clear_reentry_guard__); + return res; + } return clear(ob); } - protected override void OnSave(InterDomainContext context) + protected override void OnSave(BorrowedReference ob, InterDomainContext context) { - base.OnSave(context); - if (!this.IsClrMetaTypeInstance()) - { - BorrowedReference dict = GetObjectDict(ObjectReference); - context.Storage.AddValue("dict", PyObject.FromNullableReference(dict)); - } + base.OnSave(ob, context); + context.Storage.AddValue("impl", this); } - protected override void OnLoad(InterDomainContext context) + protected override void OnLoad(BorrowedReference ob, InterDomainContext context) { - base.OnLoad(context); - if (!this.IsClrMetaTypeInstance()) - { - var dict = context.Storage.GetValue("dict"); - SetObjectDict(ObjectReference, dict.NewReferenceOrNull().StealNullable()); - } - gcHandle = AllocGCHandle(); - SetGCHandle(ObjectReference, gcHandle); + base.OnLoad(ob, context); + var gcHandle = GCHandle.Alloc(this); + SetGCHandle(ob, gcHandle); } /// /// Implements __getitem__ for reflected classes and value types. /// - public static NewReference mp_subscript(BorrowedReference ob, BorrowedReference idx) + static NewReference mp_subscript_impl(BorrowedReference ob, BorrowedReference idx) { BorrowedReference tp = Runtime.PyObject_TYPE(ob); var cls = (ClassBase)GetManagedObject(tp)!; @@ -455,7 +437,7 @@ public static NewReference mp_subscript(BorrowedReference ob, BorrowedReference /// /// Implements __setitem__ for reflected classes and value types. /// - public static int mp_ass_subscript(BorrowedReference ob, BorrowedReference idx, BorrowedReference v) + static int mp_ass_subscript_impl(BorrowedReference ob, BorrowedReference idx, BorrowedReference v) { BorrowedReference tp = Runtime.PyObject_TYPE(ob); var cls = (ClassBase)GetManagedObject(tp)!; @@ -539,15 +521,44 @@ static IEnumerable GetCallImplementations(Type type) => type.GetMethods(BindingFlags.Public | BindingFlags.Instance) .Where(m => m.Name == "__call__"); - public virtual void InitializeSlots(SlotsHolder slotsHolder) + public virtual void InitializeSlots(BorrowedReference pyType, SlotsHolder slotsHolder) { if (!this.type.Valid) return; - if (GetCallImplementations(this.type.Value).Any() - && !slotsHolder.IsHolding(TypeOffset.tp_call)) + if (GetCallImplementations(this.type.Value).Any()) + { + TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_call, new Interop.BBB_N(tp_call_impl), slotsHolder); + } + + if (indexer is not null) + { + if (indexer.CanGet) + { + TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.mp_subscript, new Interop.BB_N(mp_subscript_impl), slotsHolder); + } + if (indexer.CanSet) + { + TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.mp_ass_subscript, new Interop.BBB_I32(mp_ass_subscript_impl), slotsHolder); + } + } + + if (typeof(IEnumerable).IsAssignableFrom(type.Value) + || typeof(IEnumerator).IsAssignableFrom(type.Value)) + { + TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_iter, new Interop.B_N(tp_iter_impl), slotsHolder); + } + + if (mp_length_slot.CanAssign(type.Value)) { - TypeManager.InitializeSlot(ObjectReference, TypeOffset.tp_call, new Interop.BBB_N(tp_call_impl), slotsHolder); + TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.mp_length, new Interop.B_P(mp_length_slot.impl), slotsHolder); } } + + protected virtual void OnDeserialization(object sender) + { + this.dotNetMembers = new List(); + } + + void IDeserializationCallback.OnDeserialization(object sender) => this.OnDeserialization(sender); } } diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index a34bd8a40..b9b3419de 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -6,10 +6,13 @@ using System.Reflection; using System.Reflection.Emit; using System.Runtime.InteropServices; -using System.Threading.Tasks; using Python.Runtime.Native; +#pragma warning disable CS0618 // Type or member is obsolete. OK for internal use +using static Python.Runtime.PythonDerivedType; +#pragma warning restore CS0618 // Type or member is obsolete + namespace Python.Runtime { /// @@ -68,22 +71,30 @@ internal ClassDerivedObject(Type tp) : base(tp) public new static void tp_dealloc(NewReference ob) { - var self = (CLRObject)GetManagedObject(ob.Borrow())!; + var self = (CLRObject?)GetManagedObject(ob.Borrow()); // don't let the python GC destroy this object - Runtime.PyObject_GC_UnTrack(self.pyHandle); - - // The python should now have a ref count of 0, but we don't actually want to - // deallocate the object until the C# object that references it is destroyed. - // So we don't call PyObject_GC_Del here and instead we set the python - // reference to a weak reference so that the C# object can be collected. - GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); - Debug.Assert(self.TypeReference == Runtime.PyObject_TYPE(self.ObjectReference)); - SetGCHandle(self.ObjectReference, self.TypeReference, gc); - self.gcHandle.Free(); - self.gcHandle = gc; + Runtime.PyObject_GC_UnTrack(ob.Borrow()); + + // self may be null after Shutdown begun + if (self is not null) + { + // The python should now have a ref count of 0, but we don't actually want to + // deallocate the object until the C# object that references it is destroyed. + // So we don't call PyObject_GC_Del here and instead we set the python + // reference to a weak reference so that the C# object can be collected. + GCHandle oldHandle = GetGCHandle(ob.Borrow()); + GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); + SetGCHandle(ob.Borrow(), gc); + oldHandle.Free(); + } } + /// + /// No-op clear. Real cleanup happens in + /// + public new static int tp_clear(BorrowedReference ob) => 0; + /// /// Called from Converter.ToPython for types that are python subclasses of managed types. /// The referenced python object is returned instead of a new wrapper. @@ -92,29 +103,37 @@ internal static NewReference ToPython(IPythonDerivedType obj) { // derived types have a __pyobj__ field that gets set to the python // object in the overridden constructor - FieldInfo fi = obj.GetType().GetField("__pyobj__"); - var self = (CLRObject)fi.GetValue(obj); + BorrowedReference self; + try + { + self = GetPyObj(obj).CheckRun(); + } catch (RuntimeShutdownException e) + { + Exceptions.SetError(e); + return default; + } - var result = new NewReference(self.ObjectReference); + var result = new NewReference(self); // when the C# constructor creates the python object it starts as a weak // reference with a reference count of 0. Now we're passing this object // to Python the reference count needs to be incremented and the reference // needs to be replaced with a strong reference to stop the C# object being // collected while Python still has a reference to it. - if (Runtime.Refcount(result.Borrow()) == 1) + if (Runtime.Refcount(self) == 1) { - Runtime._Py_NewReference(self.ObjectReference); - GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); - SetGCHandle(self.ObjectReference, self.TypeReference, gc); - self.gcHandle.Free(); - self.gcHandle = gc; + Runtime._Py_NewReference(self); + GCHandle weak = GetGCHandle(self); + var clrObject = GetManagedObject(self); + GCHandle gc = GCHandle.Alloc(clrObject, GCHandleType.Normal); + SetGCHandle(self, gc); + weak.Free(); // now the object has a python reference it's safe for the python GC to track it - Runtime.PyObject_GC_Track(self.pyHandle); + Runtime.PyObject_GC_Track(self); } - return result.AnalyzerWorkaround(); + return result; } /// @@ -126,7 +145,7 @@ internal static Type CreateDerivedType(string name, Type baseType, BorrowedReference py_dict, string namespaceStr, - string assemblyName, + string? assemblyName, string moduleName = "Python.Runtime.Dynamic.dll") { // TODO: clean up @@ -160,7 +179,11 @@ internal static Type CreateDerivedType(string name, // add a field for storing the python object pointer // FIXME: fb not used - FieldBuilder fb = typeBuilder.DefineField("__pyobj__", typeof(CLRObject), FieldAttributes.Public); + FieldBuilder fb = typeBuilder.DefineField(PyObjName, +#pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. + typeof(UnsafeReferenceWithRun), +#pragma warning restore CS0618 // Type or member is obsolete + FieldAttributes.Private); // override any constructors ConstructorInfo[] constructors = baseClass.GetConstructors(); @@ -257,7 +280,7 @@ internal static Type CreateDerivedType(string name, ILGenerator il = methodBuilder.GetILGenerator(); il.Emit(OpCodes.Ldarg_0); #pragma warning disable CS0618 // PythonDerivedType is for internal use only - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("Finalize")); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize))); #pragma warning restore CS0618 // PythonDerivedType is for internal use only il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); @@ -331,7 +354,7 @@ private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuil } il.Emit(OpCodes.Ldloc_0); #pragma warning disable CS0618 // PythonDerivedType is for internal use only - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeCtor")); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(InvokeCtor))); #pragma warning restore CS0618 // PythonDerivedType is for internal use only il.Emit(OpCodes.Ret); } @@ -636,6 +659,9 @@ private static ModuleBuilder GetModuleBuilder(string assemblyName, string module [Obsolete(Util.InternalUseOnly)] public class PythonDerivedType { + internal const string PyObjName = "__pyobj__"; + internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.NonPublic; + /// /// This is the implementation of the overridden methods in the derived /// type. It looks for a python method with the same name as the method @@ -645,24 +671,16 @@ public class PythonDerivedType /// public static T InvokeMethod(IPythonDerivedType obj, string methodName, string origMethodName, object[] args) { - FieldInfo fi = obj.GetType().GetField("__pyobj__"); - var self = (CLRObject)fi.GetValue(obj); + var self = GetPyObj(obj); - if (null != self) + if (null != self.Ref) { var disposeList = new List(); PyGILState gs = Runtime.PyGILState_Ensure(); try { - var pyself = new PyObject(self.ObjectReference); - disposeList.Add(pyself); - - Runtime.XIncref(Runtime.PyNone); - var pynone = new PyObject(Runtime.PyNone); - disposeList.Add(pynone); - - PyObject method = pyself.GetAttr(methodName, pynone); - disposeList.Add(method); + using var pyself = new PyObject(self.CheckRun()); + using PyObject method = pyself.GetAttr(methodName, Runtime.None); if (method.Reference != Runtime.PyNone) { // if the method hasn't been overridden then it will be a managed object @@ -707,24 +725,17 @@ public static T InvokeMethod(IPythonDerivedType obj, string methodName, strin public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, string origMethodName, object[] args) { - FieldInfo fi = obj.GetType().GetField("__pyobj__"); - var self = (CLRObject)fi.GetValue(obj); - if (null != self) + var self = GetPyObj(obj); + if (null != self.Ref) { var disposeList = new List(); PyGILState gs = Runtime.PyGILState_Ensure(); try { - var pyself = new PyObject(self.ObjectReference); - disposeList.Add(pyself); - - Runtime.XIncref(Runtime.PyNone); - var pynone = new PyObject(Runtime.PyNone); - disposeList.Add(pynone); - - PyObject method = pyself.GetAttr(methodName, pynone); + using var pyself = new PyObject(self.CheckRun()); + PyObject method = pyself.GetAttr(methodName, Runtime.None); disposeList.Add(method); - if (method.Reference != Runtime.PyNone) + if (method.Reference != Runtime.None) { // if the method hasn't been overridden then it will be a managed object ManagedType? managedMethod = ManagedType.GetManagedObject(method); @@ -767,10 +778,9 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s public static T InvokeGetProperty(IPythonDerivedType obj, string propertyName) { - FieldInfo fi = obj.GetType().GetField("__pyobj__"); - var self = (CLRObject)fi.GetValue(obj); + var self = GetPyObj(obj); - if (null == self) + if (null == self.Ref) { throw new NullReferenceException("Instance must be specified when getting a property"); } @@ -778,7 +788,7 @@ public static T InvokeGetProperty(IPythonDerivedType obj, string propertyName PyGILState gs = Runtime.PyGILState_Ensure(); try { - using var pyself = new PyObject(self.ObjectReference); + using var pyself = new PyObject(self.CheckRun()); using (PyObject pyvalue = pyself.GetAttr(propertyName)) { return pyvalue.As(); @@ -792,10 +802,9 @@ public static T InvokeGetProperty(IPythonDerivedType obj, string propertyName public static void InvokeSetProperty(IPythonDerivedType obj, string propertyName, T value) { - FieldInfo fi = obj.GetType().GetField("__pyobj__"); - var self = (CLRObject)fi.GetValue(obj); + var self = GetPyObj(obj); - if (null == self) + if (null == self.Ref) { throw new NullReferenceException("Instance must be specified when setting a property"); } @@ -803,7 +812,7 @@ public static void InvokeSetProperty(IPythonDerivedType obj, string propertyN PyGILState gs = Runtime.PyGILState_Ensure(); try { - using var pyself = new PyObject(self.ObjectReference); + using var pyself = new PyObject(self.CheckRun()); using var pyvalue = Converter.ToPythonImplicit(value).MoveToPyObject(); pyself.SetAttr(propertyName, pyvalue); } @@ -822,77 +831,67 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec obj, args); - CLRObject? self = null; + NewReference self = default; PyGILState gs = Runtime.PyGILState_Ensure(); try { // create the python object - var type = TypeManager.GetType(obj.GetType()); - self = new CLRObject(obj, type); + var type = ClassManager.GetClass(obj.GetType()); + self = CLRObject.GetReference(obj, type); // set __pyobj__ to self and deref the python object which will allow this // object to be collected. - FieldInfo fi = obj.GetType().GetField("__pyobj__"); - fi.SetValue(obj, self); + SetPyObj(obj, self.Borrow()); } finally { // Decrement the python object's reference count. // This doesn't actually destroy the object, it just sets the reference to this object // to be a weak reference and it will be destroyed when the C# object is destroyed. - if (null != self) + if (!self.IsNull()) { - Runtime.XDecref(self.pyHandle); + Runtime.XDecref(self.Steal()); } Runtime.PyGILState_Release(gs); } } - public static void Finalize(IPythonDerivedType obj) + public static void PyFinalize(IPythonDerivedType obj) { - FieldInfo fi = obj.GetType().GetField("__pyobj__"); - var self = (CLRObject)fi.GetValue(obj); + // the C# object is being destroyed which must mean there are no more + // references to the Python object as well + var self = GetPyObj(obj); + Finalizer.Instance.AddDerivedFinalizedObject(ref self.RawObj, self.Run); + } - // If python's been terminated then just free the gchandle. - lock (Runtime.IsFinalizingLock) - { - if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) - { - if (self.gcHandle.IsAllocated) self.gcHandle.Free(); - return; - } - } + internal static void Finalize(IntPtr derived) + { + bool deleted = CLRObject.reflectedObjects.Remove(derived); + Debug.Assert(deleted); - // delete the python object in an async task as we may not be able to acquire - // the GIL immediately and we don't want to block the GC thread. - // FIXME: t isn't used - Task t = Task.Factory.StartNew(() => - { - lock (Runtime.IsFinalizingLock) - { - // If python's been terminated then just free the gchandle. - if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) - { - if (self.gcHandle.IsAllocated) self.gcHandle.Free(); - return; - } + var @ref = NewReference.DangerousFromPointer(derived); - PyGILState gs = Runtime.PyGILState_Ensure(); - try - { - // the C# object is being destroyed which must mean there are no more - // references to the Python object as well so now we can dealloc the - // python object. - Runtime.PyObject_GC_Del(self.pyHandle); - self.gcHandle.Free(); - } - finally - { - Runtime.PyGILState_Release(gs); - } - } - }); + ClassBase.tp_clear(@ref.Borrow()); + + // rare case when it's needed + // matches correspdonging PyObject_GC_UnTrack + // in ClassDerivedObject.tp_dealloc + Runtime.PyObject_GC_Del(@ref.Steal()); + } + + internal static FieldInfo? GetPyObjField(Type type) => type.GetField(PyObjName, PyObjFlags); + + internal static UnsafeReferenceWithRun GetPyObj(IPythonDerivedType obj) + { + FieldInfo fi = GetPyObjField(obj.GetType())!; + return (UnsafeReferenceWithRun)fi.GetValue(obj); + } + + static void SetPyObj(IPythonDerivedType obj, BorrowedReference pyObj) + { + FieldInfo fi = GetPyObjField(obj.GetType())!; + fi.SetValue(obj, new UnsafeReferenceWithRun(pyObj)); } } } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index b61697390..9e15b2bd1 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -1,5 +1,4 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -34,7 +33,7 @@ internal class ClassManager BindingFlags.Public | BindingFlags.NonPublic; - private static Dictionary cache = new(capacity: 128); + internal static Dictionary cache = new(capacity: 128); private static readonly Type dtype; private ClassManager() @@ -55,58 +54,23 @@ public static void Reset() cache.Clear(); } - internal static void DisposePythonWrappersForClrTypes() + internal static void RemoveClasses() { - var visited = new HashSet(); - var visitedHandle = GCHandle.Alloc(visited); - var visitedPtr = (IntPtr)visitedHandle; - try + foreach (var @class in cache.Values) { - foreach (var cls in cache.Values) - { - // XXX: Force to release instance's managed resources - // but not dealloc itself immediately. - // These managed resources should preserve vacant shells - // since others may still referencing it. - cls.CallTypeTraverse(TraverseTypeClear, visitedPtr); - cls.CallTypeClear(); - } - } - finally - { - visitedHandle.Free(); + @class.Dispose(); } cache.Clear(); } - private static int TraverseTypeClear(BorrowedReference ob, IntPtr arg) - { - var visited = (HashSet)GCHandle.FromIntPtr(arg).Target; - if (!visited.Add(ob.DangerousGetAddressOrNull())) - { - return 0; - } - var clrObj = ManagedType.GetManagedObject(ob); - if (clrObj != null) - { - clrObj.CallTypeTraverse(TraverseTypeClear, arg); - clrObj.CallTypeClear(); - } - return 0; - } - internal static ClassManagerState SaveRuntimeData() { - var contexts = new Dictionary(PythonReferenceComparer.Instance); + var contexts = new Dictionary(); foreach (var cls in cache) { - if (!cls.Key.Valid) - { - // Don't serialize an invalid class - continue; - } - var context = contexts[cls.Value.pyHandle] = new InterDomainContext(); - cls.Value.Save(context); + var context = contexts[cls.Value] = new InterDomainContext(); + var cb = (ClassBase)ManagedType.GetManagedObject(cls.Value)!; + cb.Save(cls.Value, context); // Remove all members added in InitBaseClass. // this is done so that if domain reloads and a member of a @@ -114,11 +78,9 @@ internal static ClassManagerState SaveRuntimeData() // Python object's dictionary tool; thus raising an AttributeError // instead of a TypeError. // Classes are re-initialized on in RestoreRuntimeData. - using var dict = Runtime.PyObject_GenericGetDict(cls.Value.TypeReference); - foreach (var member in cls.Value.dotNetMembers) + using var dict = Runtime.PyObject_GenericGetDict(cls.Value); + foreach (var member in cb.dotNetMembers) { - // No need to decref the member, the ClassBase instance does - // not own the reference. if ((Runtime.PyDict_DelItemString(dict.Borrow(), member) == -1) && (Exceptions.ExceptionMatches(Exceptions.KeyError))) { @@ -132,7 +94,7 @@ internal static ClassManagerState SaveRuntimeData() } } // We modified the Type object, notify it we did. - Runtime.PyType_Modified(cls.Value.TypeReference); + Runtime.PyType_Modified(cls.Value); } return new() @@ -142,66 +104,40 @@ internal static ClassManagerState SaveRuntimeData() }; } - internal static Dictionary RestoreRuntimeData(ClassManagerState storage) + internal static void RestoreRuntimeData(ClassManagerState storage) { cache = storage.Cache; - var invalidClasses = new List>(); + var invalidClasses = new List>(); var contexts = storage.Contexts; - var loadedObjs = new Dictionary(); foreach (var pair in cache) { - if (!pair.Key.Valid) + var context = contexts[pair.Value]; + if (pair.Key.Valid) + { + pair.Value.Restore(context); + } + else { invalidClasses.Add(pair); - continue; + var cb = new UnloadedClass(pair.Key.Name); + cb.Load(pair.Value, context); + pair.Value.Restore(cb); } - // Ensure, that matching Python type exists first. - // It is required for self-referential classes - // (e.g. with members, that refer to the same class) - var pyType = InitPyType(pair.Key.Value, pair.Value); - // re-init the class - InitClassBase(pair.Key.Value, pair.Value, pyType); - // We modified the Type object, notify it we did. - Runtime.PyType_Modified(pair.Value.TypeReference); - var context = contexts[pair.Value.pyHandle]; - pair.Value.Load(context); - var slotsHolder = TypeManager.GetSlotsHolder(pyType); - pair.Value.InitializeSlots(slotsHolder); - Runtime.PyType_Modified(pair.Value.TypeReference); - loadedObjs.Add(pair.Value, context); - } - - foreach (var pair in invalidClasses) - { - cache.Remove(pair.Key); - pair.Value.pyHandle.Dispose(); } - - return loadedObjs; } /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. /// - /// A Borrowed reference to the ClassBase object - internal static ClassBase GetClass(Type type) + internal static ReflectedClrType GetClass(Type type) => ReflectedClrType.GetOrCreate(type); + + internal static ClassBase GetClassImpl(Type type) { - cache.TryGetValue(type, out var cb); - if (cb != null) - { - return cb; - } - cb = CreateClass(type); - cache.Add(type, cb); - // Ensure, that matching Python type exists first. - // It is required for self-referential classes - // (e.g. with members, that refer to the same class) - var pyType = InitPyType(type, cb); - // Initialize the object later, as this might call this GetClass method - // recursively (for example when a nested class inherits its declaring class...) - InitClassBase(type, cb, pyType); - return cb; + var pyType = GetClass(type); + var impl = (ClassBase)ManagedType.GetManagedObject(pyType)!; + Debug.Assert(impl is not null); + return impl!; } @@ -210,7 +146,7 @@ internal static ClassBase GetClass(Type type) /// managed type. The new object will be associated with a generated /// Python type object. /// - private static ClassBase CreateClass(Type type) + internal static ClassBase CreateClass(Type type) { // Next, select the appropriate managed implementation class. // Different kinds of types, such as array types or interface @@ -249,7 +185,9 @@ private static ClassBase CreateClass(Type type) impl = new ExceptionClassObject(type); } - else if (null != type.GetField("__pyobj__")) +#pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. + else if (null != PythonDerivedType.GetPyObjField(type)) +#pragma warning restore CS0618 // Type or member is obsolete { impl = new ClassDerivedObject(type); } @@ -263,17 +201,7 @@ private static ClassBase CreateClass(Type type) return impl; } - private static PyType InitPyType(Type type, ClassBase impl) - { - var pyType = TypeManager.GetOrCreateClass(type); - - // Set the handle attributes on the implementing instance. - impl.pyHandle = impl.tpHandle = pyType; - - return pyType; - } - - private static void InitClassBase(Type type, ClassBase impl, PyType pyType) + internal static void InitClassBase(Type type, ClassBase impl, PyType pyType) { // First, we introspect the managed type and build some class // information, including generating the member descriptors @@ -282,29 +210,25 @@ private static void InitClassBase(Type type, ClassBase impl, PyType pyType) ClassInfo info = GetClassInfo(type); impl.indexer = info.indexer; + impl.richcompare.Clear(); - // Now we force initialize the Python type object to reflect the given - // managed type, filling the Python type slots with thunks that - // point to the managed methods providing the implementation. - - - TypeManager.GetOrInitializeClass(impl, type); - + // Finally, initialize the class __dict__ and return the object. using var newDict = Runtime.PyObject_GenericGetDict(pyType.Reference); BorrowedReference dict = newDict.Borrow(); - - IDictionaryEnumerator iter = info.members.GetEnumerator(); - while (iter.MoveNext()) + foreach (var iter in info.members) { - var item = (ManagedType)iter.Value; - var name = (string)iter.Key; + var item = iter.Value; + var name = iter.Key; impl.dotNetMembers.Add(name); - Runtime.PyDict_SetItemString(dict, name, item.ObjectReference); - // Decref the item now that it's been used. - if (ClassBase.CilToPyOpMap.TryGetValue(name, out var pyOp)) { - impl.richcompare.Add(pyOp, (MethodObject)item); + Runtime.PyDict_SetItemString(dict, name, item); + if (ClassBase.CilToPyOpMap.TryGetValue(name, out var pyOp) + // workaround for unintialized types crashing in GetManagedObject + && item is not ReflectedClrType + && ManagedType.GetManagedObject(item) is MethodObject method) + { + impl.richcompare.Add(pyOp, method); } } @@ -330,11 +254,11 @@ private static void InitClassBase(Type type, ClassBase impl, PyType pyType) // Implement Overloads on the class object if (!CLRModule._SuppressOverloads) { - var ctors = new ConstructorBinding(type, pyType, co.binder); + using var ctors = new ConstructorBinding(type, pyType, co.binder).Alloc(); // ExtensionType types are untracked, so don't Incref() them. // TODO: deprecate __overloads__ soon... - Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.ObjectReference); - Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.ObjectReference); + Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.Borrow()); + Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.Borrow()); } // don't generate the docstring if one was already set from a DocStringAttribute. @@ -396,18 +320,16 @@ internal static bool ShouldBindEvent(EventInfo ei) private static ClassInfo GetClassInfo(Type type) { var ci = new ClassInfo(); - var methods = new Hashtable(); - ArrayList list; + var methods = new Dictionary>(); MethodInfo meth; - ManagedType ob; + ExtensionType ob; string name; - object item; Type tp; int i, n; MemberInfo[] info = type.GetMembers(BindingFlags); - var local = new Hashtable(); - var items = new ArrayList(); + var local = new HashSet(); + var items = new List(); MemberInfo m; // Loop through once to find out which names are declared @@ -416,7 +338,7 @@ private static ClassInfo GetClassInfo(Type type) m = info[i]; if (m.DeclaringType == type) { - local[m.Name] = 1; + local.Add(m.Name); } } @@ -426,7 +348,7 @@ private static ClassInfo GetClassInfo(Type type) var opsImpl = typeof(EnumOps<>).MakeGenericType(type); foreach (var op in opsImpl.GetMethods(OpsHelper.BindingFlags)) { - local[op.Name] = 1; + local.Add(op.Name); } info = info.Concat(opsImpl.GetMethods(OpsHelper.BindingFlags)).ToArray(); } @@ -435,7 +357,7 @@ private static ClassInfo GetClassInfo(Type type) for (i = 0; i < info.Length; i++) { m = info[i]; - if (local[m.Name] != null) + if (local.Contains(m.Name)) { items.Add(m); } @@ -463,7 +385,7 @@ private static ClassInfo GetClassInfo(Type type) for (n = 0; n < imembers.Length; n++) { m = imembers[n]; - if (local[m.Name] == null) + if (!local.Contains(m.Name)) { items.Add(m); } @@ -475,7 +397,7 @@ private static ClassInfo GetClassInfo(Type type) var objFlags = BindingFlags.Public | BindingFlags.Instance; foreach (var mi in typeof(object).GetMembers(objFlags)) { - if (local[mi.Name] == null) + if (!local.Contains(mi.Name) && mi is not ConstructorInfo) { items.Add(mi); } @@ -495,13 +417,11 @@ private static ClassInfo GetClassInfo(Type type) continue; } name = meth.Name; - item = methods[name]; - if (item == null) + if (!methods.TryGetValue(name, out var methodList)) { - item = methods[name] = new ArrayList(); + methodList = methods[name] = new List(); } - list = (ArrayList)item; - list.Add(meth); + methodList.Add(meth); continue; case MemberTypes.Property: @@ -527,7 +447,7 @@ private static ClassInfo GetClassInfo(Type type) } ob = new PropertyObject(pi); - ci.members[pi.Name] = ob; + ci.members[pi.Name] = ob.AllocObject(); continue; case MemberTypes.Field: @@ -537,7 +457,7 @@ private static ClassInfo GetClassInfo(Type type) continue; } ob = new FieldObject(fi); - ci.members[mi.Name] = ob; + ci.members[mi.Name] = ob.AllocObject(); continue; case MemberTypes.Event: @@ -546,8 +466,10 @@ private static ClassInfo GetClassInfo(Type type) { continue; } - ob = new EventObject(ei); - ci.members[ei.Name] = ob; + ob = ei.AddMethod.IsStatic + ? new EventBinding(ei) + : new EventObject(ei); + ci.members[ei.Name] = ob.AllocObject(); continue; case MemberTypes.NestedType: @@ -558,29 +480,20 @@ private static ClassInfo GetClassInfo(Type type) continue; } // Note the given instance might be uninitialized - ob = GetClass(tp); - if (ob.pyHandle is null && ob is ClassObject) - { - ob.pyHandle = ob.tpHandle = TypeManager.GetOrCreateClass(tp); - } - Debug.Assert(ob.pyHandle is not null); - // GetClass returns a Borrowed ref. ci.members owns the reference. - ci.members[mi.Name] = ob; + var pyType = GetClass(tp); + // make a copy, that could be disposed later + ci.members[mi.Name] = new ReflectedClrType(pyType); continue; } } - IDictionaryEnumerator iter = methods.GetEnumerator(); - - while (iter.MoveNext()) + foreach (var iter in methods) { - name = (string)iter.Key; - list = (ArrayList)iter.Value; - - var mlist = (MethodInfo[])list.ToArray(typeof(MethodInfo)); + name = iter.Key; + var mlist = iter.Value.ToArray(); ob = new MethodObject(type, name, mlist); - ci.members[name] = ob; + ci.members[name] = ob.AllocObject(); if (mlist.Any(OperatorMethod.IsOperatorMethod)) { string pyName = OperatorMethod.GetPyMethodName(name); @@ -588,10 +501,10 @@ private static ClassInfo GetClassInfo(Type type) OperatorMethod.FilterMethods(mlist, out var forwardMethods, out var reverseMethods); // Only methods where the left operand is the declaring type. if (forwardMethods.Length > 0) - ci.members[pyName] = new MethodObject(type, name, forwardMethods); + ci.members[pyName] = new MethodObject(type, name, forwardMethods).AllocObject(); // Only methods where only the right operand is the declaring type. if (reverseMethods.Length > 0) - ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods); + ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods).AllocObject(); } } @@ -624,11 +537,10 @@ private static ClassInfo GetClassInfo(Type type) private class ClassInfo { public Indexer? indexer; - public Hashtable members; + public readonly Dictionary members = new(); internal ClassInfo() { - members = new Hashtable(); indexer = null; } } diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 914c4f91f..6a5c17236 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -174,8 +174,8 @@ public override NewReference type_subscript(BorrowedReference idx) return Exceptions.RaiseTypeError("type expected"); } Type a = t.MakeArrayType(); - ClassBase o = ClassManager.GetClass(a); - return new NewReference(o.ObjectReference); + PyType o = ClassManager.GetClass(a); + return new NewReference(o); } // If there are generics in our namespace with the same base name @@ -190,7 +190,7 @@ public override NewReference type_subscript(BorrowedReference idx) Type gtype = AssemblyManager.LookupTypes($"{type.Value.FullName}`{types.Length}").FirstOrDefault(); if (gtype != null) { - var g = (GenericType)ClassManager.GetClass(gtype); + var g = (GenericType)ClassManager.GetClassImpl(gtype); return g.type_subscript(idx); } return Exceptions.RaiseTypeError("unsubscriptable object"); diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index f07d27615..f3fed3ce2 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; @@ -6,82 +7,66 @@ namespace Python.Runtime { [Serializable] [DebuggerDisplay("clrO: {inst}")] - internal class CLRObject : ManagedType + internal sealed class CLRObject : ManagedType { - internal object inst; + internal readonly object inst; - internal CLRObject(object ob, PyType tp) + // "borrowed" references + internal static readonly HashSet reflectedObjects = new(); + static NewReference Create(object ob, BorrowedReference tp) { Debug.Assert(tp != null); - using var py = Runtime.PyType_GenericAlloc(tp, 0); + var py = Runtime.PyType_GenericAlloc(tp, 0); - tpHandle = tp; - pyHandle = py.MoveToPyObject(); - inst = ob; + var self = new CLRObject(ob); - GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - InitGCHandle(ObjectReference, type: TypeReference, gc); + GCHandle gc = GCHandle.Alloc(self); + InitGCHandle(py.Borrow(), type: tp, gc); + + bool isNew = reflectedObjects.Add(py.DangerousGetAddress()); + Debug.Assert(isNew); // Fix the BaseException args (and __cause__ in case of Python 3) // slot if wrapping a CLR exception - if (ob is Exception e) Exceptions.SetArgsAndCause(ObjectReference, e); - } + if (ob is Exception e) Exceptions.SetArgsAndCause(py.Borrow(), e); - protected CLRObject() - { + return py; } - static CLRObject GetInstance(object ob, PyType pyType) + CLRObject(object inst) { - return new CLRObject(ob, pyType); - } - - - static CLRObject GetInstance(object ob) - { - ClassBase cc = ClassManager.GetClass(ob.GetType()); - return GetInstance(ob, cc.tpHandle); + this.inst = inst; } internal static NewReference GetReference(object ob, BorrowedReference pyType) - { - CLRObject co = GetInstance(ob, new PyType(pyType)); - return new NewReference(co.pyHandle); - } + => Create(ob, pyType); internal static NewReference GetReference(object ob, Type type) { - ClassBase cc = ClassManager.GetClass(type); - CLRObject co = GetInstance(ob, cc.tpHandle); - return new NewReference(co.pyHandle); + PyType cc = ClassManager.GetClass(type); + return Create(ob, cc); } - internal static NewReference GetReference(object ob) { - CLRObject co = GetInstance(ob); - return new NewReference(co.pyHandle); + PyType cc = ClassManager.GetClass(ob.GetType()); + return Create(ob, cc); } - internal static CLRObject Restore(object ob, BorrowedReference pyHandle, InterDomainContext context) + internal static void Restore(object ob, BorrowedReference pyHandle, InterDomainContext context) { - var pyObj = new PyObject(pyHandle); - CLRObject co = new CLRObject() - { - inst = ob, - pyHandle = pyObj, - tpHandle = pyObj.GetPythonType(), - }; - Debug.Assert(co.tpHandle != null); - co.Load(context); - return co; + var co = new CLRObject(ob); + co.OnLoad(pyHandle, context); } - protected override void OnLoad(InterDomainContext context) + protected override void OnLoad(BorrowedReference ob, InterDomainContext context) { - base.OnLoad(context); - GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - SetGCHandle(ObjectReference, TypeReference, gc); + base.OnLoad(ob, context); + GCHandle gc = GCHandle.Alloc(this); + SetGCHandle(ob, gc); + + bool isNew = reflectedObjects.Add(ob.DangerousGetAddress()); + Debug.Assert(isNew); } } } diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index c35a96427..780db6424 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -77,7 +77,7 @@ public static NewReference tp_descr_get(BorrowedReference op, BorrowedReference return Exceptions.RaiseTypeError("How in the world could that happen!"); } }*/ - return new NewReference(self.pyHandle); + return new NewReference(op); } /// @@ -110,7 +110,7 @@ public static NewReference mp_subscript(BorrowedReference op, BorrowedReference return Exceptions.RaiseTypeError("No match found for constructor signature"); } var boundCtor = new BoundContructor(tp, self.typeToCreate, self.ctorBinder, ci); - return new NewReference(boundCtor.pyHandle); + return boundCtor.Alloc(); } /// @@ -147,24 +147,13 @@ public static NewReference tp_repr(BorrowedReference ob) return new NewReference(self.repr); } - protected override void Clear(BorrowedReference ob) - { - Runtime.Py_CLEAR(ref this.repr); - base.Clear(ob); - } - public static int tp_traverse(BorrowedReference ob, IntPtr visit, IntPtr arg) { - var self = (ConstructorBinding)GetManagedObject(ob)!; - int res = PyVisit(self.typeToCreate, visit, arg); - if (res != 0) return res; + var self = (ConstructorBinding?)GetManagedObject(ob); + if (self is null) return 0; - if (self.repr is not null) - { - res = PyVisit(self.repr, visit, arg); - if (res != 0) return res; - } - return 0; + int res = PyVisit(self.typeToCreate, visit, arg); + return res; } } @@ -241,24 +230,13 @@ public static NewReference tp_repr(BorrowedReference ob) return new NewReference(self.repr); } - protected override void Clear(BorrowedReference ob) - { - Runtime.Py_CLEAR(ref this.repr); - base.Clear(ob); - } - public static int tp_traverse(BorrowedReference ob, IntPtr visit, IntPtr arg) { - var self = (BoundContructor)GetManagedObject(ob)!; - int res = PyVisit(self.typeToCreate, visit, arg); - if (res != 0) return res; + var self = (BoundContructor?)GetManagedObject(ob); + if (self is null) return 0; - if (self.repr is not null) - { - res = PyVisit(self.repr, visit, arg); - if (res != 0) return res; - } - return 0; + int res = PyVisit(self.typeToCreate, visit, arg); + return res; } } } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 8fbaccdf8..ff1f01a64 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -140,8 +140,8 @@ internal static NewReference ToPython(object? value, Type type) if (type.IsInterface) { - var ifaceObj = (InterfaceObject)ClassManager.GetClass(type); - return ifaceObj.WrapObject(value); + var ifaceObj = (InterfaceObject)ClassManager.GetClassImpl(type); + return ifaceObj.TryWrapObject(value); } if (type.IsArray || type.IsEnum) @@ -163,7 +163,7 @@ internal static NewReference ToPython(object? value, Type type) // pyHandle as is, do not convert. if (value is ModuleObject modobj) { - return new NewReference(modobj.ObjectReference); + throw new NotImplementedException(); } // hmm - from Python, we almost never care what the declared diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index 77318a6f9..7dfdebcbd 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -106,36 +106,34 @@ static IPyObjectEncoder[] GetEncoders(Type type) #endregion #region Decoding - static readonly ConcurrentDictionary pythonToClr = new(); - internal static bool TryDecode(BorrowedReference value, BorrowedReference type, Type targetType, out object? result) - => TryDecode(value, type.DangerousGetAddress(), targetType, out result); - internal static bool TryDecode(BorrowedReference pyHandle, IntPtr pyType, Type targetType, out object? result) + static readonly ConcurrentDictionary pythonToClr = new(); + internal static bool TryDecode(BorrowedReference pyHandle, BorrowedReference pyType, Type targetType, out object? result) { if (pyHandle == null) throw new ArgumentNullException(nameof(pyHandle)); - if (pyType == IntPtr.Zero) throw new ArgumentNullException(nameof(pyType)); + if (pyType == null) throw new ArgumentNullException(nameof(pyType)); if (targetType == null) throw new ArgumentNullException(nameof(targetType)); - var decoder = pythonToClr.GetOrAdd(new TypePair(pyType, targetType), pair => GetDecoder(pair.PyType, pair.ClrType)); + var key = new TypePair(pyType.DangerousGetAddress(), targetType); + var (_, decoder) = pythonToClr.GetOrAdd(key, pair => GetDecoder(pair.PyType, pair.ClrType)); result = null; if (decoder == null) return false; return decoder.Invoke(pyHandle, out result); } - static Converter.TryConvertFromPythonDelegate? GetDecoder(IntPtr sourceType, Type targetType) + static (PyType, Converter.TryConvertFromPythonDelegate?) GetDecoder(IntPtr sourceType, Type targetType) { - IPyObjectDecoder decoder; var sourceTypeRef = new BorrowedReference(sourceType); Debug.Assert(PyType.IsType(sourceTypeRef)); - using (var pyType = new PyType(sourceTypeRef, prevalidated: true)) + var pyType = new PyType(sourceTypeRef, prevalidated: true); + + IPyObjectDecoder decoder; + lock (decoders) { - lock (decoders) - { - decoder = decoders.GetDecoder(pyType, targetType); - if (decoder == null) return null; - } + decoder = decoders.GetDecoder(pyType, targetType); + if (decoder == null) return default; } - var decode = genericDecode.MakeGenericMethod(targetType); + var decode = genericDecode.MakeGenericMethod(targetType)!; bool TryDecode(BorrowedReference pyHandle, out object? result) { @@ -151,7 +149,9 @@ bool TryDecode(BorrowedReference pyHandle, out object? result) return success; } - return TryDecode; + // returning PyType here establishes strong reference to the object, + // that ensures the PyType we use as the converter cache key is not deallocated + return (pyType, TryDecode); } static readonly MethodInfo genericDecode = typeof(IPyObjectDecoder).GetMethod(nameof(IPyObjectDecoder.TryDecode)); diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index bccbf568a..43a75aba7 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -71,7 +71,7 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, } Delegate d = PythonEngine.DelegateManager.GetDelegate(type, new PyObject(method)); - return CLRObject.GetReference(d, self.pyHandle); + return CLRObject.GetReference(d, ClassManager.GetClass(type)); } diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 7d8630f47..69ca8f88e 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -1,4 +1,6 @@ using System; +using System.Diagnostics; +using System.Reflection; namespace Python.Runtime { @@ -8,15 +10,22 @@ namespace Python.Runtime [Serializable] internal class EventBinding : ExtensionType { - private EventObject e; + private readonly string name; + private readonly EventHandlerCollection e; private PyObject? target; - public EventBinding(EventObject e, PyObject? target) + public EventBinding(string name, EventHandlerCollection e, PyObject? target) { + this.name = name; this.target = target; this.e = e; } + public EventBinding(EventInfo @event) : this(@event.Name, new EventHandlerCollection(@event), target: null) + { + Debug.Assert(@event.AddMethod.IsStatic); + } + /// /// EventBinding += operator implementation. @@ -36,7 +45,7 @@ public static NewReference nb_inplace_add(BorrowedReference ob, BorrowedReferenc return default; } - return new NewReference(self.pyHandle); + return new NewReference(ob); } @@ -58,9 +67,13 @@ public static NewReference nb_inplace_subtract(BorrowedReference ob, BorrowedRef return default; } - return new NewReference(self.pyHandle); + return new NewReference(ob); } + /// + public static int tp_descr_set(BorrowedReference ds, BorrowedReference ob, BorrowedReference val) + => EventObject.tp_descr_set(ds, ob, val); + /// /// EventBinding __hash__ implementation. @@ -79,12 +92,7 @@ public static nint tp_hash(BorrowedReference ob) } } - nint y = Runtime.PyObject_Hash(self.e.pyHandle); - if (y == -1) - { - return y; - } - + nint y = self.e.GetHashCode(); return x ^ y; } @@ -96,14 +104,8 @@ public static NewReference tp_repr(BorrowedReference ob) { var self = (EventBinding)GetManagedObject(ob)!; string type = self.target == null ? "unbound" : "bound"; - string s = string.Format("<{0} event '{1}'>", type, self.e.name); + string s = string.Format("<{0} event '{1}'>", type, self.name); return Runtime.PyString_FromString(s); } - - protected override void Clear(BorrowedReference ob) - { - Runtime.Py_CLEAR(ref this.target); - base.Clear(ob); - } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 37eae432c..90346f2d2 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Diagnostics; using System.Reflection; namespace Python.Runtime @@ -10,124 +11,16 @@ namespace Python.Runtime [Serializable] internal class EventObject : ExtensionType { - internal string name; - internal EventBinding? unbound; - internal EventInfo info; - internal Hashtable? reg; + internal readonly string name; + internal readonly EventHandlerCollection reg; public EventObject(EventInfo info) { + Debug.Assert(!info.AddMethod.IsStatic); this.name = info.Name; - this.info = info; + this.reg = new EventHandlerCollection(info); } - - /// - /// Register a new Python object event handler with the event. - /// - internal bool AddEventHandler(BorrowedReference target, PyObject handler) - { - object? obj = null; - if (target != null) - { - var co = (CLRObject)GetManagedObject(target)!; - obj = co.inst; - } - - // Create a true delegate instance of the appropriate type to - // wrap the Python handler. Note that wrapper delegate creation - // always succeeds, though calling the wrapper may fail. - Type type = info.EventHandlerType; - Delegate d = PythonEngine.DelegateManager.GetDelegate(type, handler); - - // Now register the handler in a mapping from instance to pairs - // of (handler hash, delegate) so we can lookup to remove later. - // All this is done lazily to avoid overhead until an event is - // actually subscribed to by a Python event handler. - if (reg == null) - { - reg = new Hashtable(); - } - object key = obj ?? info.ReflectedType; - var list = reg[key] as ArrayList; - if (list == null) - { - list = new ArrayList(); - reg[key] = list; - } - list.Add(new Handler(Runtime.PyObject_Hash(handler), d)); - - // Note that AddEventHandler helper only works for public events, - // so we have to get the underlying add method explicitly. - object[] args = { d }; - MethodInfo mi = info.GetAddMethod(true); - mi.Invoke(obj, BindingFlags.Default, null, args, null); - - return true; - } - - - /// - /// Remove the given Python object event handler. - /// - internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference handler) - { - if (reg == null) - { - Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); - return false; - } - - object? obj = null; - if (target != null) - { - var co = (CLRObject)GetManagedObject(target)!; - obj = co.inst; - } - - nint hash = Runtime.PyObject_Hash(handler); - if (hash == -1 && Exceptions.ErrorOccurred()) - { - return false; - } - - object key = obj ?? info.ReflectedType; - var list = reg[key] as ArrayList; - - if (list == null) - { - Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); - return false; - } - - object?[] args = { null }; - MethodInfo mi = info.GetRemoveMethod(true); - - for (var i = 0; i < list.Count; i++) - { - var item = (Handler)list[i]; - if (item.hash != hash) - { - continue; - } - args[0] = item.del; - try - { - mi.Invoke(obj, BindingFlags.Default, null, args, null); - } - catch - { - continue; - } - list.RemoveAt(i); - return true; - } - - Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); - return false; - } - - /// /// Descriptor __get__ implementation. A getattr on an event returns /// a "bound" event that keeps a reference to the object instance. @@ -135,25 +28,15 @@ internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference han public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference ob, BorrowedReference tp) { var self = GetManagedObject(ds) as EventObject; - EventBinding binding; if (self == null) { return Exceptions.RaiseTypeError("invalid argument"); } - // If the event is accessed through its type (rather than via - // an instance) we return an 'unbound' EventBinding that will - // be cached for future accesses through the type. - if (ob == null) { - if (self.unbound == null) - { - self.unbound = new EventBinding(self, target: null); - } - binding = self.unbound; - return new NewReference(binding.pyHandle); + return new NewReference(ds); } if (Runtime.PyObject_IsInstance(ob, tp) < 1) @@ -161,8 +44,7 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference return Exceptions.RaiseTypeError("invalid argument"); } - binding = new EventBinding(self, new PyObject(ob)); - return new NewReference(binding.pyHandle); + return new EventBinding(self.name, self.reg, new PyObject(ob)).Alloc(); } @@ -195,13 +77,6 @@ public static NewReference tp_repr(BorrowedReference ob) var self = (EventObject)GetManagedObject(ob)!; return Runtime.PyString_FromString($""); } - - - protected override void Clear(BorrowedReference ob) - { - this.unbound = null!; - base.Clear(ob); - } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index bea997a3c..479e7a5d5 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -163,10 +163,13 @@ internal static void SetArgsAndCause(BorrowedReference ob, Exception e) args = Runtime.PyTuple_New(0); } - if (Runtime.PyObject_SetAttrString(ob, "args", args.Borrow()) != 0) + using (args) { - args.Dispose(); - throw PythonException.ThrowLastAsClrException(); + if (Runtime.PyObject_SetAttrString(ob, "args", args.Borrow()) != 0) + { + args.Dispose(); + throw PythonException.ThrowLastAsClrException(); + } } if (e.InnerException != null) @@ -226,15 +229,12 @@ public static bool ExceptionMatches(BorrowedReference ob) } /// - /// SetError Method - /// - /// /// Sets the current Python exception given a native string. /// This is a wrapper for the Python PyErr_SetString call. - /// - public static void SetError(BorrowedReference ob, string value) + /// + public static void SetError(BorrowedReference type, string message) { - Runtime.PyErr_SetString(ob, value); + Runtime.PyErr_SetString(type, message); } /// @@ -378,10 +378,10 @@ public static void deprecation(string message) //==================================================================== /// - /// Raises a TypeError exception and attaches any existing exception as its cause. + /// Raises a and attaches any existing exception as its cause. /// /// The exception message - /// IntPtr.Zero + /// null internal static NewReference RaiseTypeError(string message) { var cause = PythonException.FetchCurrentOrNullRaw(); diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 52d1180da..d583f6710 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; namespace Python.Runtime @@ -11,7 +13,7 @@ namespace Python.Runtime [Serializable] internal abstract class ExtensionType : ManagedType { - public ExtensionType() + public virtual NewReference Alloc() { // Create a new PyObject whose type is a generated type that is // implemented by the particular concrete ExtensionType subclass. @@ -29,50 +31,33 @@ public ExtensionType() NewReference py = Runtime.PyType_GenericAlloc(tp, 0); - // Borrowed reference. Valid as long as pyHandle is valid. - tpHandle = new PyType(tp, prevalidated: true); - pyHandle = py.MoveToPyObject(); - #if DEBUG - GetGCHandle(ObjectReference, TypeReference, out var existing); + GetGCHandle(py.BorrowOrThrow(), tp, out var existing); System.Diagnostics.Debug.Assert(existing == IntPtr.Zero); #endif - SetupGc(); + SetupGc(py.Borrow(), tp); + + return py; } - void SetupGc () + public PyObject AllocObject() => new PyObject(Alloc().Steal()); + + // "borrowed" references + internal static readonly HashSet loadedExtensions = new(); + void SetupGc (BorrowedReference ob, BorrowedReference tp) { - GCHandle gc = AllocGCHandle(TrackTypes.Extension); - InitGCHandle(ObjectReference, TypeReference, gc); + GCHandle gc = GCHandle.Alloc(this); + InitGCHandle(ob, tp, gc); + + bool isNew = loadedExtensions.Add(ob.DangerousGetAddress()); + Debug.Assert(isNew); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most // concrete extension types, so untrack the object to save calls // from Python into the managed runtime that are pure overhead. - Runtime.PyObject_GC_UnTrack(pyHandle); - } - - - protected virtual void Dealloc(NewReference lastRef) - { - var type = Runtime.PyObject_TYPE(lastRef.Borrow()); - Runtime.PyObject_GC_Del(lastRef.Steal()); - - this.FreeGCHandle(); - - // we must decref our type: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc - Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress())); - } - - /// DecRefs and nulls any fields pointing back to Python - protected virtual void Clear(BorrowedReference ob) - { - if (this.pyHandle?.IsDisposed == false) - { - ClearObjectDict(this.ObjectReference); - } - // Not necessary for decref of `tpHandle` - it is borrowed + Runtime.PyObject_GC_UnTrack(ob); } /// @@ -89,26 +74,31 @@ public static int tp_setattro(BorrowedReference ob, BorrowedReference key, Borro return -1; } - public static void tp_dealloc(NewReference lastRef) + public unsafe static void tp_dealloc(NewReference lastRef) { - // Clean up a Python instance of this extension type. This - // frees the allocated Python object and decrefs the type. - var self = (ExtensionType?)GetManagedObject(lastRef.Borrow()); - self?.Clear(lastRef.Borrow()); - self?.Dealloc(lastRef.AnalyzerWorkaround()); + Runtime.PyObject_GC_UnTrack(lastRef.Borrow()); + + tp_clear(lastRef.Borrow()); + + bool deleted = loadedExtensions.Remove(lastRef.DangerousGetAddress()); + Debug.Assert(deleted); + + // we must decref our type: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc + DecrefTypeAndFree(lastRef.Steal()); } public static int tp_clear(BorrowedReference ob) { - var self = (ExtensionType?)GetManagedObject(ob); - self?.Clear(ob); - return 0; + TryFreeGCHandle(ob); + + int res = ClassBase.BaseUnmanagedClear(ob); + return res; } - protected override void OnLoad(InterDomainContext context) + protected override void OnLoad(BorrowedReference ob, InterDomainContext context) { - base.OnLoad(context); - SetupGc(); + base.OnLoad(ob, context); + SetupGc(ob, Runtime.PyObject_TYPE(ob)); } } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 24cb9ba87..09ffe5c06 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -4,6 +4,7 @@ using System.ComponentModel; using System.Diagnostics; using System.Linq; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -40,7 +41,8 @@ public ErrorArgs(Exception error) [DefaultValue(true)] public bool Enable { get; set; } = true; - private ConcurrentQueue _objQueue = new (); + private ConcurrentQueue _objQueue = new(); + private readonly ConcurrentQueue _derivedQueue = new(); private int _throttled; #region FINALIZER_CHECK @@ -110,7 +112,7 @@ internal void ThrottledCollect() if (!started) throw new InvalidOperationException($"{nameof(PythonEngine)} is not initialized"); _throttled = unchecked(this._throttled + 1); - if (!Enable || _throttled < Threshold) return; + if (!started || !Enable || _throttled < Threshold) return; _throttled = 0; this.Collect(); } @@ -120,7 +122,11 @@ internal List GetCollectedObjects() return _objQueue.Select(o => o.PyObj).ToList(); } - internal void AddFinalizedObject(ref IntPtr obj, int run) + internal void AddFinalizedObject(ref IntPtr obj, int run +#if TRACE_ALLOC + , StackTrace stackTrace +#endif + ) { Debug.Assert(obj != IntPtr.Zero); if (!Enable) @@ -128,15 +134,37 @@ internal void AddFinalizedObject(ref IntPtr obj, int run) return; } + Debug.Assert(Runtime.Refcount(new BorrowedReference(obj)) > 0); + #if FINALIZER_CHECK lock (_queueLock) #endif { - this._objQueue.Enqueue(new PendingFinalization { PyObj = obj, RuntimeRun = run }); + this._objQueue.Enqueue(new PendingFinalization { + PyObj = obj, RuntimeRun = run, +#if TRACE_ALLOC + StackTrace = stackTrace.ToString(), +#endif + }); } obj = IntPtr.Zero; } + internal void AddDerivedFinalizedObject(ref IntPtr derived, int run) + { + if (derived == IntPtr.Zero) + throw new ArgumentNullException(nameof(derived)); + + if (!Enable) + { + return; + } + + var pending = new PendingFinalization { PyObj = derived, RuntimeRun = run }; + derived = IntPtr.Zero; + _derivedQueue.Enqueue(pending); + } + internal static void Initialize() { Instance.started = true; @@ -148,8 +176,13 @@ internal static void Shutdown() Instance.started = false; } - private void DisposeAll() + internal nint DisposeAll() { + if (_objQueue.IsEmpty && _derivedQueue.IsEmpty) + return 0; + + nint collected = 0; + BeforeCollect?.Invoke(this, new CollectArgs() { ObjectCount = _objQueue.Count @@ -162,6 +195,7 @@ private void DisposeAll() ValidateRefCount(); #endif Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); + Debug.Assert(errType.IsNull()); int run = Runtime.GetRun(); @@ -180,6 +214,7 @@ private void DisposeAll() IntPtr copyForException = obj.PyObj; Runtime.XDecref(StolenReference.Take(ref obj.PyObj)); + collected++; try { Runtime.CheckExceptionOccurred(); @@ -189,6 +224,24 @@ private void DisposeAll() HandleFinalizationException(obj.PyObj, e); } } + + while (!_derivedQueue.IsEmpty) + { + if (!_derivedQueue.TryDequeue(out var derived)) + continue; + + if (derived.RuntimeRun != run) + { + HandleFinalizationException(derived.PyObj, new RuntimeShutdownException(derived.PyObj)); + continue; + } + +#pragma warning disable CS0618 // Type or member is obsolete. OK for internal use + PythonDerivedType.Finalize(derived.PyObj); +#pragma warning restore CS0618 // Type or member is obsolete + + collected++; + } } finally { @@ -197,6 +250,7 @@ private void DisposeAll() Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); } } + return collected; } void HandleFinalizationException(IntPtr obj, Exception cause) @@ -285,7 +339,11 @@ private void ValidateRefCount() struct PendingFinalization { public IntPtr PyObj; + public BorrowedReference Ref => new(PyObj); public int RuntimeRun; +#if TRACE_ALLOC + public string StackTrace; +#endif } public class FinalizationException : Exception diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 62ee66c38..b40fa2cd6 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -14,10 +14,11 @@ internal static class ImportHook { #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. // set in Initialize - private static CLRModule root; + private static PyObject root; + private static CLRModule clrModule; private static PyModule py_clr_module; #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - static BorrowedReference ClrModuleReference => py_clr_module.Reference; + internal static BorrowedReference ClrModuleReference => py_clr_module.Reference; private const string LoaderCode = @" import importlib.abc @@ -58,14 +59,14 @@ def find_spec(klass, fullname, paths=None, target=None): internal static unsafe void Initialize() { // Initialize the clr module and tell Python about it. - root = new CLRModule(); + root = CLRModule.Create(out clrModule).MoveToPyObject(); // create a python module with the same methods as the clr module-like object py_clr_module = new PyModule(Runtime.PyModule_New("clr").StealOrThrow()); // both dicts are borrowed references BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root); Runtime.PyDict_Update(mod_dict, clr_dict.BorrowOrThrow()); BorrowedReference dict = Runtime.PyImport_GetModuleDict(); @@ -88,7 +89,7 @@ internal static void Shutdown() TeardownNameSpaceTracking(); Runtime.Py_CLEAR(ref py_clr_module!); - root.pyHandle.Dispose(); + root.Dispose(); root = null!; CLRModule.Reset(); } @@ -98,6 +99,7 @@ private static Dictionary GetDotNetModules() BorrowedReference pyModules = Runtime.PyImport_GetModuleDict(); using var items = Runtime.PyDict_Items(pyModules); nint length = Runtime.PyList_Size(items.BorrowOrThrow()); + Debug.Assert(length >= 0); var modules = new Dictionary(); for (nint i = 0; i < length; i++) { @@ -116,7 +118,7 @@ internal static ImportHookState SaveRuntimeData() return new() { PyCLRModule = py_clr_module, - Root = root.pyHandle, + Root = new PyObject(root), Modules = GetDotNetModules(), }; } @@ -139,7 +141,8 @@ internal static void RestoreRuntimeData(ImportHookState storage) { py_clr_module = storage.PyCLRModule; var rootHandle = storage.Root; - root = (CLRModule)ManagedType.GetManagedObject(rootHandle)!; + root = new PyObject(rootHandle); + clrModule = (CLRModule)ManagedType.GetManagedObject(rootHandle)!; BorrowedReference dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); SetupNamespaceTracking(); @@ -152,6 +155,7 @@ static void SetupImportHook() // Create the import hook module using var import_hook_module = Runtime.PyModule_New("clr.loader"); BorrowedReference mod_dict = Runtime.PyModule_GetDict(import_hook_module.BorrowOrThrow()); + Debug.Assert(mod_dict != null); // Run the python code to create the module's classes. var builtins = Runtime.PyEval_GetBuiltins(); @@ -195,7 +199,7 @@ static void SetupNamespaceTracking() throw PythonException.ThrowLastAsClrException(); } } - if (Runtime.PyDict_SetItemString(root.dict, _available_namespaces, newset.Borrow()) != 0) + if (Runtime.PyDict_SetItemString(clrModule.dict, _available_namespaces, newset.Borrow()) != 0) { throw PythonException.ThrowLastAsClrException(); } @@ -207,7 +211,7 @@ static void SetupNamespaceTracking() static void TeardownNameSpaceTracking() { // If the C# runtime isn't loaded, then there are no namespaces available - Runtime.PyDict_SetItemString(root.dict, _available_namespaces, Runtime.PyNone); + Runtime.PyDict_SetItemString(clrModule.dict, _available_namespaces, Runtime.PyNone); } static readonly ConcurrentQueue addPending = new(); @@ -227,7 +231,7 @@ internal static int AddPendingNamespaces() internal static void AddNamespaceWithGIL(string name) { using var pyNs = Runtime.PyString_FromString(name); - var nsSet = Runtime.PyDict_GetItemString(root.dict, _available_namespaces); + var nsSet = Runtime.PyDict_GetItemString(clrModule.dict, _available_namespaces); if (!(nsSet.IsNull || nsSet == Runtime.PyNone)) { if (Runtime.PySet_Add(nsSet, pyNs.BorrowOrThrow()) != 0) @@ -244,12 +248,12 @@ internal static void AddNamespaceWithGIL(string name) /// internal static void UpdateCLRModuleDict() { - root.InitializePreload(); + clrModule.InitializePreload(); // update the module dictionary with the contents of the root dictionary - root.LoadNames(); + clrModule.LoadNames(); BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root); Runtime.PyDict_Update(py_mod_dict, clr_dict.BorrowOrThrow()); } @@ -279,29 +283,29 @@ public static PyObject Import(string modname) // setting clr.preload = True ModuleObject? head = null; - ModuleObject tail = root; - root.InitializePreload(); + ModuleObject tail = clrModule; + clrModule.InitializePreload(); string[] names = modname.Split('.'); foreach (string name in names) { - ManagedType mt = tail.GetAttribute(name, true); - if (!(mt is ModuleObject)) + using var nested = tail.GetAttribute(name, true); + if (nested.IsNull() || ManagedType.GetManagedObject(nested.Borrow()) is not ModuleObject module) { Exceptions.SetError(Exceptions.ImportError, $"'{name}' Is not a ModuleObject."); throw PythonException.ThrowLastAsClrException(); } if (head == null) { - head = (ModuleObject)mt; + head = module; } - tail = (ModuleObject)mt; + tail = module; if (CLRModule.preload) { tail.LoadNames(); } } - return new PyObject(tail.ObjectReference); + return tail.Alloc().MoveToPyObject(); } } } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 891653fcb..4903b6f76 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -113,7 +113,7 @@ internal NewReference GetDefaultArgs(BorrowedReference args) using var arg = Converter.ToPython(pi[i + pynargs].DefaultValue, pi[i + pynargs].ParameterType); Runtime.PyTuple_SetItem(defaultArgs.Borrow(), i, arg.Steal()); } - return defaultArgs.AnalyzerWorkaround(); + return defaultArgs; } } } diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 0cc396cef..f71f78236 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -76,14 +76,17 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, return default; } - return self.WrapObject(obj); + return self.TryWrapObject(obj); } /// /// Wrap the given object in an interface object, so that only methods /// of the interface are available. /// - public NewReference WrapObject(object impl) => CLRObject.GetReference(impl, pyHandle); + public NewReference TryWrapObject(object impl) + => this.type.Valid + ? CLRObject.GetReference(impl, ClassManager.GetClass(this.type.Value)) + : Exceptions.RaiseTypeError(this.type.DeletedMessage); /// /// Expose the wrapped implementation through attributes in both diff --git a/src/runtime/intern_.cs b/src/runtime/intern_.cs index edb3340c5..09986593e 100644 --- a/src/runtime/intern_.cs +++ b/src/runtime/intern_.cs @@ -4,37 +4,39 @@ namespace Python.Runtime { static class PyIdentifier { - static IntPtr f__name__; + static IntPtr f__name__; public static BorrowedReference __name__ => new(f__name__); - static IntPtr f__dict__; + static IntPtr f__dict__; public static BorrowedReference __dict__ => new(f__dict__); - static IntPtr f__doc__; + static IntPtr f__doc__; public static BorrowedReference __doc__ => new(f__doc__); - static IntPtr f__class__; + static IntPtr f__class__; public static BorrowedReference __class__ => new(f__class__); - static IntPtr f__module__; + static IntPtr f__clear_reentry_guard__; + public static BorrowedReference __clear_reentry_guard__ => new(f__clear_reentry_guard__); + static IntPtr f__module__; public static BorrowedReference __module__ => new(f__module__); - static IntPtr f__file__; + static IntPtr f__file__; public static BorrowedReference __file__ => new(f__file__); - static IntPtr f__slots__; + static IntPtr f__slots__; public static BorrowedReference __slots__ => new(f__slots__); - static IntPtr f__self__; + static IntPtr f__self__; public static BorrowedReference __self__ => new(f__self__); - static IntPtr f__annotations__; + static IntPtr f__annotations__; public static BorrowedReference __annotations__ => new(f__annotations__); - static IntPtr f__init__; + static IntPtr f__init__; public static BorrowedReference __init__ => new(f__init__); - static IntPtr f__repr__; + static IntPtr f__repr__; public static BorrowedReference __repr__ => new(f__repr__); - static IntPtr f__import__; + static IntPtr f__import__; public static BorrowedReference __import__ => new(f__import__); - static IntPtr f__builtins__; + static IntPtr f__builtins__; public static BorrowedReference __builtins__ => new(f__builtins__); - static IntPtr fbuiltins; + static IntPtr fbuiltins; public static BorrowedReference builtins => new(fbuiltins); - static IntPtr f__overloads__; + static IntPtr f__overloads__; public static BorrowedReference __overloads__ => new(f__overloads__); - static IntPtr fOverloads; + static IntPtr fOverloads; public static BorrowedReference Overloads => new(fOverloads); } @@ -47,6 +49,7 @@ static partial class InternString "__dict__", "__doc__", "__class__", + "__clear_reentry_guard__", "__module__", "__file__", "__slots__", diff --git a/src/runtime/intern_.tt b/src/runtime/intern_.tt index d867bab35..bb8d9f12d 100644 --- a/src/runtime/intern_.tt +++ b/src/runtime/intern_.tt @@ -7,6 +7,7 @@ "__dict__", "__doc__", "__class__", + "__clear_reentry_guard__", "__module__", "__file__", "__slots__", @@ -34,7 +35,7 @@ namespace Python.Runtime foreach (var name in internNames) { #> - static IntPtr f<#= name #>; + static IntPtr f<#= name #>; public static BorrowedReference <#= name #> => new(f<#= name #>); <# } diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 2d35d28a0..e894fa591 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -81,7 +81,6 @@ public enum TypeFlags: int HasClrInstance = (1 << 15), /// PythonNet specific Subclass = (1 << 16), - HaveIndex = (1 << 17), /* Objects support nb_index in PyNumberMethods */ HaveVersionTag = (1 << 18), ValidVersionTag = (1 << 19), @@ -218,21 +217,6 @@ public ThunkInfo(Delegate target) } } - [StructLayout(LayoutKind.Sequential)] - struct PyGC_Node - { - public IntPtr gc_next; - public IntPtr gc_prev; - public IntPtr gc_refs; - } - - [StructLayout(LayoutKind.Sequential)] - struct PyGC_Head - { - public PyGC_Node gc; - } - - [StructLayout(LayoutKind.Sequential)] struct PyMethodDef { diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 08255dc42..d7472cc61 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -14,105 +14,10 @@ namespace Python.Runtime [Serializable] internal abstract class ManagedType { - internal enum TrackTypes - { - Untrack, - Extension, - Wrapper, - } - - [NonSerialized] - internal GCHandle gcHandle; // Native handle - - internal PyObject pyHandle; // PyObject * - internal PyType tpHandle; // PyType * - - [NonSerialized] - internal bool clearReentryGuard; - - internal BorrowedReference ObjectReference - { - get - { - Debug.Assert(pyHandle != null); - return pyHandle.Reference; - } - } - - internal BorrowedReference TypeReference - { - get - { - Debug.Assert(tpHandle != null); - return tpHandle.Reference; - } - } - - private static readonly Dictionary _managedObjs = new Dictionary(); - - internal long RefCount - { - get - { - var gs = Runtime.PyGILState_Ensure(); - try - { - return Runtime.Refcount(pyHandle); - } - finally - { - Runtime.PyGILState_Release(gs); - } - } - } - - internal GCHandle AllocGCHandle(TrackTypes track = TrackTypes.Untrack) - { - gcHandle = GCHandle.Alloc(this); - if (track != TrackTypes.Untrack) - { - _managedObjs.Add(this, track); - } - return gcHandle; - } - - internal void FreeGCHandle() - { - _managedObjs.Remove(this); - if (gcHandle.IsAllocated) - { - gcHandle.Free(); - gcHandle = default; - } - } - /// /// Given a Python object, return the associated managed object or null. /// internal static ManagedType? GetManagedObject(BorrowedReference ob) - { - if (ob != null) - { - BorrowedReference tp = Runtime.PyObject_TYPE(ob); - if (tp == Runtime.PyTypeType || tp == Runtime.PyCLRMetaType) - { - tp = ob; - } - - var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.HasClrInstance) != 0) - { - var gc = TryGetGCHandle(ob); - return (ManagedType?)gc?.Target; - } - } - return null; - } - - /// - /// Given a Python object, return the associated managed object type or null. - /// - internal static ManagedType? GetManagedObjectType(BorrowedReference ob) { if (ob != null) { @@ -120,8 +25,8 @@ internal void FreeGCHandle() var flags = PyType.GetFlags(tp); if ((flags & TypeFlags.HasClrInstance) != 0) { - var gc = GetGCHandle(tp, Runtime.CLRMetaType); - return (ManagedType)gc.Target; + var gc = TryGetGCHandle(ob, tp); + return (ManagedType?)gc?.Target; } } return null; @@ -154,26 +59,11 @@ internal static BorrowedReference GetUnmanagedBaseType(BorrowedReference managed do { managedType = PyType.GetBase(managedType); + Debug.Assert(managedType != null); } while (IsManagedType(managedType)); return managedType; } - public bool IsClrMetaTypeInstance() - { - Debug.Assert(Runtime.PyCLRMetaType != null); - return Runtime.PyObject_TYPE(ObjectReference) == Runtime.PyCLRMetaType; - } - - internal static IDictionary GetManagedObjects() - { - return _managedObjs; - } - - internal static void ClearTrackedObjects() - { - _managedObjs.Clear(); - } - internal unsafe static int PyVisit(BorrowedReference ob, IntPtr visit, IntPtr arg) { if (ob == null) @@ -184,61 +74,53 @@ internal unsafe static int PyVisit(BorrowedReference ob, IntPtr visit, IntPtr ar return visitFunc(ob, arg); } - /// - /// Wrapper for calling tp_clear - /// - internal unsafe int CallTypeClear() + internal static unsafe void DecrefTypeAndFree(StolenReference ob) { - if (tpHandle == BorrowedReference.Null || pyHandle == BorrowedReference.Null) - { - return 0; - } + if (ob == null) throw new ArgumentNullException(nameof(ob)); + var borrowed = new BorrowedReference(ob.DangerousGetAddress()); - var clearPtr = Runtime.PyType_GetSlot(TypeReference, TypeSlotID.tp_clear); - if (clearPtr == IntPtr.Zero) - { - return 0; - } - var clearFunc = (delegate* unmanaged[Cdecl])clearPtr; - return clearFunc(pyHandle); + var type = Runtime.PyObject_TYPE(borrowed); + + var freePtr = Util.ReadIntPtr(type, TypeOffset.tp_free); + Debug.Assert(freePtr != IntPtr.Zero); + var free = (delegate* unmanaged[Cdecl])freePtr; + free(ob); + + Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress())); } + internal static int CallClear(BorrowedReference ob) + => CallTypeClear(ob, Runtime.PyObject_TYPE(ob)); + /// - /// Wrapper for calling tp_traverse + /// Wrapper for calling tp_clear /// - internal unsafe int CallTypeTraverse(Interop.BP_I32 visitproc, IntPtr arg) + internal static unsafe int CallTypeClear(BorrowedReference ob, BorrowedReference tp) { - if (tpHandle == BorrowedReference.Null || pyHandle == BorrowedReference.Null) - { - return 0; - } - var traversePtr = Runtime.PyType_GetSlot(TypeReference, TypeSlotID.tp_traverse); - if (traversePtr == IntPtr.Zero) + if (ob == null) throw new ArgumentNullException(nameof(ob)); + if (tp == null) throw new ArgumentNullException(nameof(tp)); + + var clearPtr = Util.ReadIntPtr(tp, TypeOffset.tp_clear); + if (clearPtr == IntPtr.Zero) { return 0; } - var traverseFunc = (delegate* unmanaged[Cdecl])traversePtr; - var visiPtr = Marshal.GetFunctionPointerForDelegate(visitproc); - return traverseFunc(pyHandle, visiPtr, arg); - } - - protected void TypeClear() - { - ClearObjectDict(ObjectReference); + var clearFunc = (delegate* unmanaged[Cdecl])clearPtr; + return clearFunc(ob); } - internal void Save(InterDomainContext context) + internal void Save(BorrowedReference ob, InterDomainContext context) { - OnSave(context); + OnSave(ob, context); } - internal void Load(InterDomainContext context) + internal void Load(BorrowedReference ob, InterDomainContext context) { - OnLoad(context); + OnLoad(ob, context); } - protected virtual void OnSave(InterDomainContext context) { } - protected virtual void OnLoad(InterDomainContext context) { } + protected virtual void OnSave(BorrowedReference ob, InterDomainContext context) { } + protected virtual void OnLoad(BorrowedReference ob, InterDomainContext context) { } protected static void ClearObjectDict(BorrowedReference ob) { @@ -272,7 +154,7 @@ protected static void SetObjectDictNullable(BorrowedReference ob, StolenReferenc internal static void GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, out IntPtr handle) { Debug.Assert(reflectedClrObject != null); - Debug.Assert(IsManagedType(type) || type == Runtime.CLRMetaType); + Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); int gcHandleOffset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); @@ -311,6 +193,7 @@ internal static void SetGCHandle(BorrowedReference reflectedClrObject, BorrowedR { Debug.Assert(type != null); Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); int offset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); @@ -321,6 +204,29 @@ internal static void SetGCHandle(BorrowedReference reflectedClrObject, BorrowedR internal static void SetGCHandle(BorrowedReference reflectedClrObject, GCHandle newHandle) => SetGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), newHandle); + internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject) + => TryFreeGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject)); + + internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + { + Debug.Assert(type != null); + Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int offset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); + Debug.Assert(offset > 0); + + IntPtr raw = Util.ReadIntPtr(reflectedClrObject, offset); + if (raw == IntPtr.Zero) return false; + + var handle = (GCHandle)raw; + handle.Free(); + + Util.WriteIntPtr(reflectedClrObject, offset, IntPtr.Zero); + return true; + } + internal static class Offsets { static Offsets() diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index f3a6ad469..f4ad5a4b1 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -11,7 +12,7 @@ namespace Python.Runtime /// types. It also provides support for single-inheritance from reflected /// managed types. /// - internal class MetaType : ManagedType + internal sealed class MetaType : ManagedType { #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. // set in Initialize @@ -36,12 +37,7 @@ public static PyType Initialize() public static void Release() { - if (Runtime.Refcount(PyCLRMetaType) > 1) - { - _metaSlotsHodler.ResetSlots(); - } PyCLRMetaType.Dispose(); - _metaSlotsHodler = null!; } internal static MetatypeState SaveRuntimeData() => new() { CLRMetaType = PyCLRMetaType }; @@ -166,15 +162,17 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, // derived types must have their GCHandle at the same offset as the base types int clrInstOffset = Util.ReadInt32(base_type, Offsets.tp_clr_inst_offset); + Debug.Assert(clrInstOffset > 0 + && clrInstOffset < Util.ReadInt32(type.Borrow(), TypeOffset.tp_basicsize)); Util.WriteInt32(type.Borrow(), Offsets.tp_clr_inst_offset, clrInstOffset); // for now, move up hidden handle... - IntPtr gc = Util.ReadIntPtr(base_type, Offsets.tp_clr_inst); - Util.WriteIntPtr(type.Borrow(), Offsets.tp_clr_inst, gc); + var gc = (GCHandle)Util.ReadIntPtr(base_type, Offsets.tp_clr_inst); + Util.WriteIntPtr(type.Borrow(), Offsets.tp_clr_inst, (IntPtr)GCHandle.Alloc(gc.Target)); Runtime.PyType_Modified(type.Borrow()); - return type.AnalyzerWorkaround(); + return type; } @@ -289,29 +287,31 @@ public static void tp_dealloc(NewReference lastRef) { // Fix this when we dont cheat on the handle for subclasses! - var flags = (TypeFlags)Util.ReadCLong(lastRef.Borrow(), TypeOffset.tp_flags); + var flags = PyType.GetFlags(lastRef.Borrow()); if ((flags & TypeFlags.Subclass) == 0) { - GetGCHandle(lastRef.Borrow()).Free(); + TryGetGCHandle(lastRef.Borrow())?.Free(); #if DEBUG // prevent ExecutionEngineException in debug builds in case we have a bug // this would allow using managed debugger to investigate the issue - SetGCHandle(lastRef.Borrow(), Runtime.CLRMetaType, default); + SetGCHandle(lastRef.Borrow(), default); #endif } - var op = Util.ReadIntPtr(lastRef.Borrow(), TypeOffset.ob_type); - // We must decref our type. - // type_dealloc from PyType will use it to get tp_free so we must keep the value - Runtime.XDecref(StolenReference.DangerousFromPointer(op)); + var op = Runtime.PyObject_TYPE(lastRef.Borrow()); + Debug.Assert(Runtime.PyCLRMetaType is null || Runtime.PyCLRMetaType == op); + var builtinType = Runtime.PyObject_TYPE(Runtime.PyObject_TYPE(op)); // Delegate the rest of finalization the Python metatype. Note // that the PyType_Type implementation of tp_dealloc will call // tp_free on the type of the type being deallocated - in this // case our CLR metatype. That is why we implement tp_free. - - IntPtr tp_dealloc = Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_dealloc); + IntPtr tp_dealloc = Util.ReadIntPtr(builtinType, TypeOffset.tp_dealloc); NativeCall.CallDealloc(tp_dealloc, lastRef.Steal()); + + // We must decref our type. + // type_dealloc from PyType will use it to get tp_free so we must keep the value + Runtime.XDecref(StolenReference.DangerousFromPointer(op.DangerousGetAddress())); } private static NewReference DoInstanceCheck(BorrowedReference tp, BorrowedReference args, bool checkType) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 613e80411..b2d23ab01 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -49,7 +49,7 @@ public static NewReference mp_subscript(BorrowedReference tp, BorrowedReference } var mb = new MethodBinding(self.m, self.target, self.targetType) { info = mi }; - return new NewReference(mb.pyHandle); + return mb.Alloc(); } PyObject Signature @@ -136,7 +136,7 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k case "__overloads__": case "Overloads": var om = new OverloadMapper(self.m, self.target); - return new NewReference(om.pyHandle); + return om.Alloc(); case "__signature__" when Runtime.InspectModule is not null: var sig = self.Signature; if (sig is null) @@ -263,12 +263,7 @@ public static nint tp_hash(BorrowedReference ob) } } - nint y = Runtime.PyObject_Hash(self.m.pyHandle); - if (y == -1) - { - return y; - } - + nint y = self.m.GetHashCode(); return x ^ y; } @@ -282,12 +277,5 @@ public static NewReference tp_repr(BorrowedReference ob) string name = self.m.name; return Runtime.PyString_FromString($"<{type} method '{name}'>"); } - - protected override void Clear(BorrowedReference ob) - { - this.target = null; - this.targetType = null!; - base.Clear(ob); - } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 6daa973f2..14e26c86d 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -21,7 +21,7 @@ internal class MethodObject : ExtensionType private MethodInfo[]? _info = null; private readonly List infoList; internal string name; - internal MethodBinding? unbound; + internal PyObject? unbound; internal readonly MethodBinder binder; internal bool is_static = false; @@ -157,7 +157,6 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference ob, BorrowedReference tp) { var self = (MethodObject)GetManagedObject(ds)!; - MethodBinding binding; // If the method is accessed through its type (rather than via // an instance) we return an 'unbound' MethodBinding that will @@ -167,10 +166,9 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference { if (self.unbound is null) { - self.unbound = new MethodBinding(self, target: null, targetType: new PyType(tp)); + self.unbound = new PyObject(new MethodBinding(self, target: null, targetType: new PyType(tp)).Alloc().Steal()); } - binding = self.unbound; - return new NewReference(binding.pyHandle); + return new NewReference(self.unbound); } if (Runtime.PyObject_IsInstance(ob, tp) < 1) @@ -188,13 +186,11 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference && obj.inst is IPythonDerivedType && self.type.IsInstanceOfType(obj.inst)) { - ClassBase basecls = ClassManager.GetClass(self.type); - binding = new MethodBinding(self, new PyObject(ob), basecls.pyHandle); - return new NewReference(binding.pyHandle); + var basecls = ClassManager.GetClass(self.type); + return new MethodBinding(self, new PyObject(ob), basecls).Alloc(); } - binding = new MethodBinding(self, target: new PyObject(ob), targetType: new PyType(tp)); - return new NewReference(binding.pyHandle); + return new MethodBinding(self, target: new PyObject(ob), targetType: new PyType(tp)).Alloc(); } /// @@ -205,13 +201,5 @@ public static NewReference tp_repr(BorrowedReference ob) var self = (MethodObject)GetManagedObject(ob)!; return Runtime.PyString_FromString($""); } - - protected override void Clear(BorrowedReference ob) - { - Runtime.Py_CLEAR(ref this.doc); - this.unbound = null; - ClearObjectDict(this.pyHandle); - base.Clear(ob); - } } } diff --git a/src/runtime/module.cs b/src/runtime/module.cs index 7ba9159a1..159fc6912 100644 --- a/src/runtime/module.cs +++ b/src/runtime/module.cs @@ -2,9 +2,11 @@ using System.Linq; using System.Collections.Generic; using System.Dynamic; +using System.Runtime.Serialization; namespace Python.Runtime { + [Serializable] public class PyModule : PyObject { internal BorrowedReference variables => VarsRef; @@ -41,6 +43,9 @@ internal PyModule(in StolenReference reference) : base(reference) } } + protected PyModule(SerializationInfo info, StreamingContext context) + : base(info, context) { } + private void InitializeBuiltins() { int res = Runtime.PyDict_SetItem( diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 63a97da51..21435264a 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.IO; using System.Reflection; @@ -13,10 +14,10 @@ namespace Python.Runtime [Serializable] internal class ModuleObject : ExtensionType { - private Dictionary cache; + private readonly Dictionary cache = new(); internal string moduleName; - internal readonly PyDict dict; + internal PyDict dict; protected string _namespace; private readonly PyList __all__ = new (); @@ -25,41 +26,59 @@ internal class ModuleObject : ExtensionType static readonly HashSet settableAttributes = new () {"__spec__", "__file__", "__name__", "__path__", "__loader__", "__package__"}; - public ModuleObject(string name) +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + /// is initialized in + protected ModuleObject(string name) +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. { if (name == string.Empty) { throw new ArgumentException("Name must not be empty!"); } moduleName = name; - cache = new Dictionary(); _namespace = name; + } + + internal static NewReference Create(string name) => new ModuleObject(name).Alloc(); + + public override NewReference Alloc() + { + var py = base.Alloc(); - // Use the filename from any of the assemblies just so there's something for - // anything that expects __file__ to be set. - var filename = "unknown"; - var docstring = "Namespace containing types from the following assemblies:\n\n"; - foreach (Assembly a in AssemblyManager.GetAssemblies(name)) + if (dict is null) { - if (!a.IsDynamic && a.Location != null) + // Use the filename from any of the assemblies just so there's something for + // anything that expects __file__ to be set. + var filename = "unknown"; + var docstring = "Namespace containing types from the following assemblies:\n\n"; + foreach (Assembly a in AssemblyManager.GetAssemblies(moduleName)) { - filename = a.Location; + if (!a.IsDynamic && a.Location != null) + { + filename = a.Location; + } + docstring += "- " + a.FullName + "\n"; } - docstring += "- " + a.FullName + "\n"; - } - using var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference); - dict = new PyDict(dictRef.StealOrThrow()); - using var pyname = Runtime.PyString_FromString(moduleName); - using var pyfilename = Runtime.PyString_FromString(filename); - using var pydocstring = Runtime.PyString_FromString(docstring); - BorrowedReference pycls = TypeManager.GetTypeReference(GetType()); - Runtime.PyDict_SetItem(dict, PyIdentifier.__name__, pyname.Borrow()); - Runtime.PyDict_SetItem(dict, PyIdentifier.__file__, pyfilename.Borrow()); - Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, pydocstring.Borrow()); - Runtime.PyDict_SetItem(dict, PyIdentifier.__class__, pycls); + using var dictRef = Runtime.PyObject_GenericGetDict(py.Borrow()); + dict = new PyDict(dictRef.StealOrThrow()); + using var pyname = Runtime.PyString_FromString(moduleName); + using var pyfilename = Runtime.PyString_FromString(filename); + using var pydocstring = Runtime.PyString_FromString(docstring); + BorrowedReference pycls = TypeManager.GetTypeReference(GetType()); + Runtime.PyDict_SetItem(dict, PyIdentifier.__name__, pyname.Borrow()); + Runtime.PyDict_SetItem(dict, PyIdentifier.__file__, pyfilename.Borrow()); + Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, pydocstring.Borrow()); + Runtime.PyDict_SetItem(dict, PyIdentifier.__class__, pycls); + } + else + { + SetObjectDict(py.Borrow(), new NewReference(dict).Steal()); + } InitializeModuleMembers(); + + return py; } @@ -69,16 +88,14 @@ public ModuleObject(string name) /// namespace (or null if the name is not found). This method does /// not increment the Python refcount of the returned object. /// - public ManagedType? GetAttribute(string name, bool guess) + public NewReference GetAttribute(string name, bool guess) { cache.TryGetValue(name, out var cached); if (cached != null) { - return cached; + return new NewReference(cached); } - ModuleObject m; - ClassBase c; Type type; //if (AssemblyManager.IsValidNamespace(name)) @@ -100,8 +117,8 @@ public ModuleObject(string name) // a new ModuleObject representing that namespace. if (AssemblyManager.IsValidNamespace(qname)) { - m = new ModuleObject(qname); - StoreAttribute(name, m); + var m = ModuleObject.Create(qname); + this.StoreAttribute(name, m.Borrow()); return m; } @@ -111,9 +128,9 @@ public ModuleObject(string name) type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - c = ClassManager.GetClass(type); + var c = ClassManager.GetClass(type); StoreAttribute(name, c); - return c; + return new NewReference(c); } // We didn't find the name, so we may need to see if there is a @@ -125,37 +142,31 @@ public ModuleObject(string name) // enough to complicate the implementation for now. if (guess) { - string gname = GenericUtil.GenericNameForBaseName(_namespace, name); + string gname = GenericUtil.GenericNameForBaseName(this._namespace, name); if (gname != null) { - ManagedType? o = GetAttribute(gname, false); - if (o != null) + var o = this.GetAttribute(gname, false); + if (!o.IsNull()) { - StoreAttribute(name, o); + this.StoreAttribute(name, o.Borrow()); return o; } } } - return null; - } - - static void ImportWarning(Exception exception) - { - Exceptions.warn(exception.ToString(), Exceptions.ImportWarning); + return default; } - /// /// Stores an attribute in the instance dict for future lookups. /// - private void StoreAttribute(string name, ManagedType ob) + private void StoreAttribute(string name, BorrowedReference ob) { - if (Runtime.PyDict_SetItemString(dict, name, ob.ObjectReference) != 0) + if (Runtime.PyDict_SetItemString(dict, name, ob) != 0) { throw PythonException.ThrowLastAsClrException(); } - cache[name] = ob; + cache[name] = new PyObject(ob); } @@ -180,7 +191,8 @@ public void LoadNames() continue; } - if(GetAttribute(name, true) != null) + using var attrVal = GetAttribute(name, true); + if (!attrVal.IsNull()) { // if it's a valid attribute, add it to __all__ using var pyname = Runtime.PyString_FromString(name); @@ -217,8 +229,8 @@ internal void InitializeModuleMembers() string name = method.Name; var mi = new MethodInfo[1]; mi[0] = method; - var m = new ModuleFunctionObject(type, name, mi, allow_threads); - StoreAttribute(name, m); + using var m = new ModuleFunctionObject(type, name, mi, allow_threads).Alloc(); + StoreAttribute(name, m.Borrow()); } } @@ -229,8 +241,8 @@ internal void InitializeModuleMembers() if (attrs.Length > 0) { string name = property.Name; - var p = new ModulePropertyObject(property); - StoreAttribute(name, p); + using var p = new ModulePropertyObject(property).Alloc(); + StoreAttribute(name, p.Borrow()); } } type = type.BaseType; @@ -254,6 +266,8 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k return default; } + Debug.Assert(!self.dict.IsDisposed); + BorrowedReference op = Runtime.PyDict_GetItem(self.dict, key); if (op != null) { @@ -272,7 +286,7 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k return new NewReference(self.__all__); } - ManagedType? attr; + NewReference attr; try { @@ -286,13 +300,13 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k } - if (attr == null) + if (attr.IsNull()) { Exceptions.SetError(Exceptions.AttributeError, name); return default; } - return new NewReference(attr.ObjectReference); + return attr; } /// @@ -306,28 +320,20 @@ public static NewReference tp_repr(BorrowedReference ob) public static int tp_traverse(BorrowedReference ob, IntPtr visit, IntPtr arg) { - var self = (ModuleObject)GetManagedObject(ob)!; + var self = (ModuleObject?)GetManagedObject(ob); + if (self is null) return 0; + + Debug.Assert(self.dict == GetObjectDict(ob)); int res = PyVisit(self.dict, visit, arg); if (res != 0) return res; foreach (var attr in self.cache.Values) { - res = PyVisit(attr.ObjectReference, visit, arg); + res = PyVisit(attr, visit, arg); if (res != 0) return res; } return 0; } - protected override void Clear(BorrowedReference ob) - { - this.dict.Dispose(); - if (this.pyHandle?.IsDisposed == false) - { - ClearObjectDict(this.ObjectReference); - } - this.cache.Clear(); - base.Clear(ob); - } - /// /// Override the setattr implementation. /// This is needed because the import mechanics need @@ -338,7 +344,7 @@ protected override void Clear(BorrowedReference ob) { var managedKey = Runtime.GetManagedString(key); if ((settableAttributes.Contains(managedKey)) || - (ManagedType.GetManagedObject(val)?.GetType() == typeof(ModuleObject)) ) + (ManagedType.GetManagedObject(val) is ModuleObject) ) { var self = (ModuleObject)ManagedType.GetManagedObject(ob)!; return Runtime.PyDict_SetItem(self.dict, key, val); @@ -347,10 +353,10 @@ protected override void Clear(BorrowedReference ob) return ExtensionType.tp_setattro(ob, key, val); } - protected override void OnSave(InterDomainContext context) + protected override void OnSave(BorrowedReference ob, InterDomainContext context) { - base.OnSave(context); - System.Diagnostics.Debug.Assert(dict == GetObjectDict(ObjectReference)); + base.OnSave(ob, context); + System.Diagnostics.Debug.Assert(dict == GetObjectDict(ob)); // destroy the cache(s) foreach (var pair in cache) { @@ -370,10 +376,10 @@ protected override void OnSave(InterDomainContext context) cache.Clear(); } - protected override void OnLoad(InterDomainContext context) + protected override void OnLoad(BorrowedReference ob, InterDomainContext context) { - base.OnLoad(context); - SetObjectDict(pyHandle, new NewReference(dict).Steal()); + base.OnLoad(ob, context); + SetObjectDict(ob, new NewReference(dict).Steal()); } } @@ -385,7 +391,6 @@ protected override void OnLoad(InterDomainContext context) [Serializable] internal class CLRModule : ModuleObject { - protected static bool hacked = false; protected static bool interactive_preload = true; internal static bool preload; // XXX Test performance of new features // @@ -397,26 +402,19 @@ static CLRModule() Reset(); } - public CLRModule() : base("clr") + private CLRModule() : base("clr") { _namespace = string.Empty; + } - // This hackery is required in order to allow a plain Python to - // import the managed runtime via the CLR bootstrapper module. - // The standard Python machinery in control at the time of the - // import requires the module to pass PyModule_Check. :( - if (!hacked) - { - BorrowedReference mro = Util.ReadRef(TypeReference, TypeOffset.tp_mro); - using var ext = Runtime.ExtendTuple(mro, Runtime.PyModuleType); - Util.WriteRef(TypeReference, TypeOffset.tp_mro, ext.Steal()); - hacked = true; - } + internal static NewReference Create(out CLRModule module) + { + module = new CLRModule(); + return module.Alloc(); } public static void Reset() { - hacked = false; interactive_preload = true; preload = false; diff --git a/src/runtime/native/PyMemberFlags.cs b/src/runtime/native/PyMemberFlags.cs new file mode 100644 index 000000000..56ba8962b --- /dev/null +++ b/src/runtime/native/PyMemberFlags.cs @@ -0,0 +1,14 @@ +using System; + +namespace Python.Runtime.Native; + +[Flags] +enum PyMemberFlags: int +{ + None = 0, + ReadOnly = 1, + ReadRestricted = 2, + WriteRestricted = 4, + Restricted = (ReadRestricted | WriteRestricted), + AuditRead = ReadRestricted, +} diff --git a/src/runtime/native/PyMemberType.cs b/src/runtime/native/PyMemberType.cs new file mode 100644 index 000000000..261d552a5 --- /dev/null +++ b/src/runtime/native/PyMemberType.cs @@ -0,0 +1,38 @@ +namespace Python.Runtime.Native; + +enum PyMemberType: int +{ + Short = 0, + Int = 1, + Long = 2, + Float = 3, + Double = 4, + String = 5, + Object = 6, + /// 1-character string + Char = 7, + /// 8-bit signed int + Byte = 8, + + UByte = 9, + UShort = 10, + UInt = 11, + ULong = 12, + + StringInPlace = 13, + + /// bools contained in the structure (assumed char) + Bool = 14, + + /// + /// Like but raises AttributeError + /// when the value is NULL, instead of converting to None + /// + ObjectEx = 16, + + LongLong = 17, + ULongLong = 18, + + PySignedSizeT = 19, + AlwaysNone = 20, +} diff --git a/src/runtime/native/PyMethodFlags.cs b/src/runtime/native/PyMethodFlags.cs new file mode 100644 index 000000000..5c270871f --- /dev/null +++ b/src/runtime/native/PyMethodFlags.cs @@ -0,0 +1,38 @@ +using System; + +namespace Python.Runtime.Native; + +[Flags] +enum PyMethodFlags : int +{ + [Obsolete] + OLDARGS = 0, + VarArgs = 1, + Keywords = 2, + NoArgs = 4, + O = 8, + + Class = 0x10, + Static = 0x20, + + /// + /// Allows a method to be entered even though a slot has + /// already filled the entry. When defined, the flag allows a separate + /// method, "__contains__" for example, to coexist with a defined + /// slot like sq_contains. + /// + Coexist = 0x40, + + /// 3.10+ + FastCall = 0x80, + + /// + /// The function stores an + /// additional reference to the class that defines it; + /// both self and class are passed to it. + /// It uses PyCMethodObject instead of PyCFunctionObject. + /// May not be combined with METH_NOARGS, METH_O, METH_CLASS or METH_STATIC. + /// + /// 3.9+ + Method = 0x0200, +} diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index a3b4f4a24..c880db842 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -115,6 +115,9 @@ public static int GetSlotOffset(string slotName) return SlotOffsets[slotName]; } + public static string? GetSlotName(int offset) + => SlotOffsets.FirstOrDefault(kv => kv.Value == offset).Key; + static readonly HashSet slotNames = new HashSet(); internal static bool IsSupportedSlotName(string name) => slotNames.Contains(name); diff --git a/src/runtime/operatormethod.cs b/src/runtime/operatormethod.cs index eb9e7949a..a807f59f3 100644 --- a/src/runtime/operatormethod.cs +++ b/src/runtime/operatormethod.cs @@ -154,7 +154,7 @@ private static PyObject GetOperatorType() // A hack way for getting typeobject.c::slotdefs string code = GenerateDummyCode(); // The resulting OperatorMethod class is stored in a PyDict. - PythonEngine.Exec(code, null, locals.Handle); + PythonEngine.Exec(code, null, locals); // Return the class itself, which is a type. return locals.GetItem("OperatorMethod"); } diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index e7bb4d6d7..c75d38574 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -43,7 +43,7 @@ public static NewReference mp_subscript(BorrowedReference tp, BorrowedReference } var mb = new MethodBinding(self.m, self.target) { info = mi }; - return new NewReference(mb.pyHandle); + return mb.Alloc(); } /// @@ -54,12 +54,5 @@ public static NewReference tp_repr(BorrowedReference op) var self = (OverloadMapper)GetManagedObject(op)!; return self.m.GetDocString(); } - - protected override void Clear(BorrowedReference ob) - { - this.target = null; - this.m = null!; - base.Clear(ob); - } } } diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs index ab5a38a7f..60aeaf0b9 100644 --- a/src/runtime/pybuffer.cs +++ b/src/runtime/pybuffer.cs @@ -240,7 +240,11 @@ private void Dispose(bool disposing) if (_view.obj != IntPtr.Zero) { - Finalizer.Instance.AddFinalizedObject(ref _view.obj, _exporter.run); + Finalizer.Instance.AddFinalizedObject(ref _view.obj, _exporter.run +#if TRACE_ALLOC + , _exporter.Traceback +#endif + ); } Dispose(false); diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 033dcd169..80b8c8c9f 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -8,6 +9,7 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/dict.html /// for details. /// + [Serializable] public class PyDict : PyIterable { internal PyDict(BorrowedReference reference) : base(reference) { } @@ -32,6 +34,9 @@ public PyDict() : base(Runtime.PyDict_New().StealOrThrow()) { } } } + protected PyDict(SerializationInfo info, StreamingContext context) + : base(info, context) { } + /// /// IsDictType Method @@ -166,5 +171,15 @@ public void Clear() { Runtime.PyDict_Clear(obj); } + + public override int GetHashCode() => rawPtr.GetHashCode(); + + public override bool Equals(PyObject? other) + { + if (other is null) return false; + if (obj == other.obj) return true; + if (other is PyDict || IsDictType(other)) return base.Equals(other); + return false; + } } } diff --git a/src/runtime/pyfloat.cs b/src/runtime/pyfloat.cs index bcf39748f..7fb9e8f4d 100644 --- a/src/runtime/pyfloat.cs +++ b/src/runtime/pyfloat.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -70,6 +71,9 @@ public PyFloat(string value) : base(FromString(value)) { } + protected PyFloat(SerializationInfo info, StreamingContext context) + : base(info, context) { } + /// /// IsFloatType Method diff --git a/src/runtime/pyint.cs b/src/runtime/pyint.cs index f163681b0..d503c15f3 100644 --- a/src/runtime/pyint.cs +++ b/src/runtime/pyint.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -134,6 +135,9 @@ public PyInt(string value) : base(Runtime.PyLong_FromString(value, 0).StealOrThr { } + protected PyInt(SerializationInfo info, StreamingContext context) + : base(info, context) { } + /// /// IsIntType Method diff --git a/src/runtime/pyiter.cs b/src/runtime/pyiter.cs index 5e78cf6dd..f9847b11c 100644 --- a/src/runtime/pyiter.cs +++ b/src/runtime/pyiter.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -89,5 +90,17 @@ public void Reset() public PyObject Current => _current ?? throw new InvalidOperationException(); object System.Collections.IEnumerator.Current => Current; + + protected PyIter(SerializationInfo info, StreamingContext context) + : base(info, context) + { + _current = (PyObject?)info.GetValue("c", typeof(PyObject)); + } + + protected override void GetObjectData(SerializationInfo info, StreamingContext context) + { + base.GetObjectData(info, context); + info.AddValue("c", _current); + } } } diff --git a/src/runtime/pyiterable.cs b/src/runtime/pyiterable.cs index 4e53e3158..1a154cb54 100644 --- a/src/runtime/pyiterable.cs +++ b/src/runtime/pyiterable.cs @@ -1,13 +1,17 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Runtime.Serialization; namespace Python.Runtime { + [Serializable] public class PyIterable : PyObject, IEnumerable { internal PyIterable(BorrowedReference reference) : base(reference) { } internal PyIterable(in StolenReference reference) : base(reference) { } + protected PyIterable(SerializationInfo info, StreamingContext context) + : base(info, context) { } /// /// Creates new instance from an existing object. diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index 5abfdb621..1f0a30a23 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -9,6 +10,7 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/list.html /// for details. /// + [Serializable] public class PyList : PySequence { internal PyList(in StolenReference reference) : base(reference) { } @@ -18,6 +20,10 @@ internal PyList(in StolenReference reference) : base(reference) { } internal PyList(BorrowedReference reference) : base(reference) { } + protected PyList(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + private static BorrowedReference FromObject(PyObject o) { if (o == null || !IsListType(o)) @@ -162,5 +168,15 @@ public void Sort() throw PythonException.ThrowLastAsClrException(); } } + + public override int GetHashCode() => rawPtr.GetHashCode(); + + public override bool Equals(PyObject? other) + { + if (other is null) return false; + if (obj == other.obj) return true; + if (other is PyList || IsListType(other)) return base.Equals(other); + return false; + } } } diff --git a/src/runtime/pynumber.cs b/src/runtime/pynumber.cs index 442be230e..8754e132f 100644 --- a/src/runtime/pynumber.cs +++ b/src/runtime/pynumber.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -15,6 +16,8 @@ public class PyNumber : PyObject { internal PyNumber(in StolenReference reference) : base(reference) { } internal PyNumber(BorrowedReference reference) : base(reference) { } + protected PyNumber(SerializationInfo info, StreamingContext context) + : base(info, context) { } /// /// IsNumberType Method diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 821d115b2..894fff329 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -18,13 +18,13 @@ namespace Python.Runtime /// [Serializable] [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] - public partial class PyObject : DynamicObject, IDisposable + public partial class PyObject : DynamicObject, IDisposable, ISerializable { #if TRACE_ALLOC /// /// Trace stack for PyObject's construction /// - public StackTrace Traceback { get; private set; } + public StackTrace Traceback { get; } = new StackTrace(1); #endif protected internal IntPtr rawPtr = IntPtr.Zero; @@ -51,9 +51,6 @@ internal PyObject(IntPtr ptr) rawPtr = ptr; Finalizer.Instance.ThrottledCollect(); -#if TRACE_ALLOC - Traceback = new StackTrace(1); -#endif } [Obsolete("for testing purposes only")] @@ -64,9 +61,6 @@ internal PyObject(IntPtr ptr, bool skipCollect) rawPtr = ptr; if (!skipCollect) Finalizer.Instance.ThrottledCollect(); -#if TRACE_ALLOC - Traceback = new StackTrace(1); -#endif } /// @@ -80,9 +74,15 @@ internal PyObject(BorrowedReference reference) rawPtr = new NewReference(reference).DangerousMoveToPointer(); Finalizer.Instance.ThrottledCollect(); -#if TRACE_ALLOC - Traceback = new StackTrace(1); -#endif + } + + internal PyObject(BorrowedReference reference, bool skipCollect) + { + if (reference.IsNull) throw new ArgumentNullException(nameof(reference)); + + rawPtr = new NewReference(reference).DangerousMoveToPointer(); + if (!skipCollect) + Finalizer.Instance.ThrottledCollect(); } internal PyObject(in StolenReference reference) @@ -91,9 +91,6 @@ internal PyObject(in StolenReference reference) rawPtr = reference.DangerousGetAddressOrNull(); Finalizer.Instance.ThrottledCollect(); -#if TRACE_ALLOC - Traceback = new StackTrace(1); -#endif } // Ensure that encapsulated Python object is decref'ed appropriately @@ -109,7 +106,11 @@ internal PyObject(in StolenReference reference) Interlocked.Increment(ref Runtime._collected); - Finalizer.Instance.AddFinalizedObject(ref rawPtr, run); + Finalizer.Instance.AddFinalizedObject(ref rawPtr, run +#if TRACE_ALLOC + , Traceback +#endif + ); } Dispose(false); @@ -185,41 +186,35 @@ protected virtual void Dispose(bool disposing) return; } - if (Runtime.Py_IsInitialized() == 0) + if (Runtime.Py_IsInitialized() == 0 && Runtime._Py_IsFinalizing() != true) + { throw new InvalidOperationException("Python runtime must be initialized"); + } - CheckRun(); + nint refcount = Runtime.Refcount(this.obj); + Debug.Assert(refcount > 0, "Object refcount is 0 or less"); - if (!Runtime.IsFinalizing) + if (refcount == 1) { - long refcount = Runtime.Refcount(this.obj); - Debug.Assert(refcount > 0, "Object refcount is 0 or less"); + Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); - if (refcount == 1) + try { - Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); - - try - { - Runtime.XDecref(StolenReference.Take(ref rawPtr)); - Runtime.CheckExceptionOccurred(); - } - finally - { - // Python requires finalizers to preserve exception: - // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation - Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); - } + Runtime.XDecref(StolenReference.Take(ref rawPtr)); + Runtime.CheckExceptionOccurred(); } - else + finally { - Runtime.XDecref(StolenReference.Take(ref rawPtr)); + // Python requires finalizers to preserve exception: + // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); } } else { - throw new InvalidOperationException("Runtime is already finalizing"); + Runtime.XDecref(StolenReference.Take(ref rawPtr)); } + this.rawPtr = IntPtr.Zero; } @@ -235,6 +230,13 @@ public void Dispose() { GC.SuppressFinalize(this); Dispose(true); + + } + + internal StolenReference Steal() + { + GC.SuppressFinalize(this); + return StolenReference.Take(ref this.rawPtr); } [Obsolete("Test use only")] @@ -263,8 +265,8 @@ internal BorrowedReference GetPythonTypeReference() /// public PyType GetPythonType() { - var tp = Runtime.PyObject_TYPE(Reference); - return new PyType(tp, prevalidated: true); + var tp = Runtime.PyObject_Type(Reference); + return new PyType(tp.StealOrThrow(), prevalidated: true); } @@ -282,6 +284,8 @@ public bool TypeCheck(PyType typeOrClass) return Runtime.PyObject_TypeCheck(obj, typeOrClass.obj); } + internal PyType PyType => this.GetPythonType(); + /// /// HasAttr Method @@ -1063,17 +1067,17 @@ public PyList Dir() /// Return true if this object is equal to the given object. This /// method is based on Python equality semantics. /// - public override bool Equals(object o) + public override bool Equals(object o) => Equals(o as PyObject); + + public virtual bool Equals(PyObject? other) { - if (!(o is PyObject)) - { - return false; - } - if (obj == ((PyObject)o).obj) + if (other is null) return false; + + if (obj == other.obj) { return true; } - int r = Runtime.PyObject_Compare(obj, ((PyObject)o).obj); + int r = Runtime.PyObject_Compare(this, other); if (Exceptions.ErrorOccurred()) { throw PythonException.ThrowLastAsClrException(); @@ -1092,7 +1096,12 @@ public override bool Equals(object o) /// public override int GetHashCode() { - return ((ulong)Runtime.PyObject_Hash(obj)).GetHashCode(); + nint pyHash = Runtime.PyObject_Hash(obj); + if (pyHash == -1 && Exceptions.ErrorOccurred()) + { + throw PythonException.ThrowLastAsClrException(); + } + return pyHash.GetHashCode(); } /// @@ -1448,15 +1457,21 @@ public override IEnumerable GetDynamicMemberNames() } } - [OnSerialized] - void OnSerialized(StreamingContext context) + void ISerializable.GetObjectData(SerializationInfo info, StreamingContext context) + => GetObjectData(info, context); + protected virtual void GetObjectData(SerializationInfo info, StreamingContext context) { -#warning check that these methods are inherited properly - new NewReference(this, canBeNull: true).Steal(); +#pragma warning disable CS0618 // Type or member is obsolete + Runtime.XIncref(this); +#pragma warning restore CS0618 // Type or member is obsolete + info.AddValue("h", rawPtr.ToInt64()); + info.AddValue("r", run); } - [OnDeserialized] - void OnDeserialized(StreamingContext context) + + protected PyObject(SerializationInfo info, StreamingContext context) { + rawPtr = (IntPtr)info.GetInt64("h"); + run = info.GetInt32("r"); if (IsDisposed) GC.SuppressFinalize(this); } } diff --git a/src/runtime/pysequence.cs b/src/runtime/pysequence.cs index f3eb7cc3b..5d7417be2 100644 --- a/src/runtime/pysequence.cs +++ b/src/runtime/pysequence.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -9,10 +10,12 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/sequence.html /// for details. /// + [Serializable] public class PySequence : PyIterable { internal PySequence(BorrowedReference reference) : base(reference) { } internal PySequence(in StolenReference reference) : base(reference) { } + protected PySequence(SerializationInfo info, StreamingContext context) : base(info, context) { } /// /// Creates new instance from an existing object. diff --git a/src/runtime/pystring.cs b/src/runtime/pystring.cs index 648d5227a..cdd45e2c3 100644 --- a/src/runtime/pystring.cs +++ b/src/runtime/pystring.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -11,11 +12,12 @@ namespace Python.Runtime /// /// 2011-01-29: ...Then why does the string constructor call PyUnicode_FromUnicode()??? /// + [Serializable] public class PyString : PySequence { internal PyString(in StolenReference reference) : base(reference) { } internal PyString(BorrowedReference reference) : base(reference) { } - + protected PyString(SerializationInfo info, StreamingContext context) : base(info, context) { } private static BorrowedReference FromObject(PyObject o) { diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 814f2a86a..f3b7fa770 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using System.Runtime.Serialization; using System.Threading; using Python.Runtime.Native; @@ -222,8 +223,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } // Load the clr.py resource into the clr module - NewReference clr = Python.Runtime.ImportHook.GetCLRModule(); - BorrowedReference clr_dict = Runtime.PyModule_GetDict(clr.Borrow()); + BorrowedReference clr_dict = Runtime.PyModule_GetDict(ImportHook.ClrModuleReference); var locals = new PyDict(); try @@ -282,7 +282,7 @@ static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, s Assembly assembly = Assembly.GetExecutingAssembly(); string pyCode = assembly.ReadStringResource(resourceName); - Exec(pyCode, module_globals.DangerousGetAddress(), module_globals.DangerousGetAddress()); + Exec(pyCode, module_globals, module_globals); Runtime.PyDict_SetItemString(targetModuleDict, memberName!, module); } @@ -559,11 +559,9 @@ public static PyObject Compile(string code, string filename = "", RunFlagType mo /// Evaluate a Python expression and returns the result. /// It's a subset of Python eval function. /// - public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals = null) + public static PyObject Eval(string code, PyDict? globals = null, PyObject? locals = null) { - var globalsRef = new BorrowedReference(globals.GetValueOrDefault()); - var localsRef = new BorrowedReference(locals.GetValueOrDefault()); - PyObject result = RunString(code, globalsRef, localsRef, RunFlagType.Eval); + PyObject result = RunString(code, globals.BorrowNullable(), locals.BorrowNullable(), RunFlagType.Eval); return result; } @@ -575,11 +573,9 @@ public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals /// Run a string containing Python code. /// It's a subset of Python exec function. /// - public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = null) + public static void Exec(string code, PyDict? globals = null, PyObject? locals = null) { - var globalsRef = new BorrowedReference(globals.GetValueOrDefault()); - var localsRef = new BorrowedReference(locals.GetValueOrDefault()); - using PyObject result = RunString(code, globalsRef, localsRef, RunFlagType.File); + using PyObject result = RunString(code, globals.BorrowNullable(), locals.BorrowNullable(), RunFlagType.File); if (result.obj != Runtime.PyNone) { throw PythonException.ThrowLastAsClrException(); @@ -632,9 +628,9 @@ public static int Interrupt(ulong pythonThreadID) /// Use Exec/Eval/RunSimpleString instead. /// [Obsolete("RunString is deprecated and will be removed. Use Exec/Eval/RunSimpleString instead.")] - public static PyObject RunString(string code, IntPtr? globals = null, IntPtr? locals = null) + public static PyObject RunString(string code, PyDict? globals = null, PyObject? locals = null) { - return RunString(code, new BorrowedReference(globals.GetValueOrDefault()), new BorrowedReference(locals.GetValueOrDefault()), RunFlagType.File); + return RunString(code, globals.BorrowNullable(), locals.BorrowNullable(), RunFlagType.File); } /// @@ -751,6 +747,12 @@ public override void Dispose() public class KeywordArguments : PyDict { + public KeywordArguments() : base() + { + } + + protected KeywordArguments(SerializationInfo info, StreamingContext context) + : base(info, context) { } } public static KeywordArguments kw(params object?[] kv) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 79dc5f153..813d0e586 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -3,6 +3,8 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Runtime.Serialization; +using System.Security.Permissions; using System.Text; using Python.Runtime.Native; @@ -13,6 +15,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// + [Serializable] public class PythonException : System.Exception { public PythonException(PyType type, PyObject? value, PyObject? traceback, @@ -36,6 +39,7 @@ public PythonException(PyType type, PyObject? value, PyObject? traceback) /// It is recommended to call this as throw ThrowLastAsClrException() /// to assist control flow checks. /// + [DebuggerHidden] internal static Exception ThrowLastAsClrException() { // prevent potential interop errors in this method @@ -185,18 +189,24 @@ private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference return new PythonException(type, value, traceback, inner); } - private static Exception? TryDecodePyErr(BorrowedReference typeRef, BorrowedReference valRef, BorrowedReference tbRef) + private static PyDict ToPyErrArgs(BorrowedReference typeRef, BorrowedReference valRef, BorrowedReference tbRef) { using var type = PyType.FromReference(typeRef); using var value = PyObject.FromNullableReference(valRef); using var traceback = PyObject.FromNullableReference(tbRef); - using var errorDict = new PyDict(); - if (typeRef != null) errorDict["type"] = type; - if (valRef != null) errorDict["value"] = value ?? PyObject.None; - if (tbRef != null) errorDict["traceback"] = traceback ?? PyObject.None; + var errorDict = new PyDict(); + errorDict["type"] = type; + if (value is not null) errorDict["value"] = value; + if (traceback is not null) errorDict["traceback"] = traceback; + + return errorDict; + } + private static Exception? TryDecodePyErr(BorrowedReference typeRef, BorrowedReference valRef, BorrowedReference tbRef) + { using var pyErrType = Runtime.InteropModule.GetAttr("PyErr"); + using var errorDict = ToPyErrArgs(typeRef, valRef, tbRef); using var pyErrInfo = pyErrType.Invoke(new PyTuple(), errorDict); if (PyObjectConversions.TryDecode(pyErrInfo.Reference, pyErrType.Reference, typeof(Exception), out object? decoded) && decoded is Exception decodedPyErrInfo) @@ -254,12 +264,8 @@ private static string TracebackToString(PyObject traceback) } /// Restores python error. - public void Restore() + internal void Restore() { - CheckRuntimeIsRunning(); - - using var _ = new Py.GILState(); - NewReference type = Type.NewReferenceOrNull(); NewReference value = Value.NewReferenceOrNull(); NewReference traceback = Traceback.NewReferenceOrNull(); @@ -394,6 +400,32 @@ public PythonException Clone() => new PythonException(type: Type, value: Value, traceback: Traceback, Message, InnerException); + #region Serializable + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + protected PythonException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + Type = (PyType)info.GetValue(nameof(Type), typeof(PyType)); + Value = (PyObject)info.GetValue(nameof(Value), typeof(PyObject)); + Traceback = (PyObject)info.GetValue(nameof(Traceback), typeof(PyObject)); + } + + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + if (info == null) + { + throw new ArgumentNullException(nameof(info)); + } + + base.GetObjectData(info, context); + + info.AddValue(nameof(Type), Type); + info.AddValue(nameof(Value), Value); + info.AddValue(nameof(Traceback), Traceback); + } + #endregion + internal bool Is(BorrowedReference type) { return Runtime.PyErr_GivenExceptionMatches( @@ -416,6 +448,7 @@ internal static bool CurrentMatches(BorrowedReference ob) return Runtime.PyErr_ExceptionMatches(ob) != 0; } + [DebuggerHidden] internal static void ThrowIfIsNull(in NewReference ob) { if (ob.BorrowNullable() == null) @@ -426,6 +459,7 @@ internal static void ThrowIfIsNull(in NewReference ob) internal static BorrowedReference ThrowIfIsNull(BorrowedReference ob) => Exceptions.ErrorCheck(ob); + [DebuggerHidden] internal static void ThrowIfIsNotZero(int value) { if (value != 0) diff --git a/src/runtime/pytuple.cs b/src/runtime/pytuple.cs index e2bca2bf7..6e212a808 100644 --- a/src/runtime/pytuple.cs +++ b/src/runtime/pytuple.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -20,6 +21,7 @@ internal PyTuple(in StolenReference reference) : base(reference) { } /// The object reference is not checked for type-correctness. /// internal PyTuple(BorrowedReference reference) : base(reference) { } + protected PyTuple(SerializationInfo info, StreamingContext context) : base(info, context) { } private static BorrowedReference FromObject(PyObject o) { diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs index dd35b92a8..260800592 100644 --- a/src/runtime/pytype.cs +++ b/src/runtime/pytype.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.Runtime.Serialization; using Python.Runtime.Native; @@ -34,6 +35,8 @@ internal PyType(in StolenReference reference, bool prevalidated = false) : base( throw new ArgumentException("object is not a type"); } + protected PyType(SerializationInfo info, StreamingContext context) : base(info, context) { } + internal new static PyType? FromNullableReference(BorrowedReference reference) => reference == null ? null @@ -63,6 +66,10 @@ internal TypeFlags Flags set => Util.WriteCLong(this, TypeOffset.tp_flags, (long)value); } + internal PyDict Dict => new(Util.ReadRef(this, TypeOffset.tp_dict)); + + internal PyTuple MRO => new(GetMRO(this)); + /// Checks if specified object is a Python type. public static bool IsType(PyObject value) { @@ -86,7 +93,7 @@ public static PyType Get(Type clrType) throw new ArgumentNullException(nameof(clrType)); } - return new PyType(TypeManager.GetType(clrType)); + return new PyType(ClassManager.GetClass(clrType)); } internal BorrowedReference BaseReference diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2cc5aabcd..2f1d36ac6 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -54,10 +54,6 @@ private static string GetDefaultDllName(Version version) return prefix + "python" + suffix + ext; } - // set to true when python is finalizing - internal static object IsFinalizingLock = new object(); - internal static bool IsFinalizing; - private static bool _isInitialized = false; internal static readonly bool Is32Bit = IntPtr.Size == 4; @@ -155,7 +151,6 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; - IsFinalizing = false; Finalizer.Initialize(); InitPyMembers(); @@ -178,7 +173,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } else { - PyCLRMetaType = MetaType.Initialize(); // Steal a reference + PyCLRMetaType = MetaType.Initialize(); ImportHook.Initialize(); } Exceptions.Initialize(); @@ -214,13 +209,12 @@ private static void InitPyMembers() SetPyMember(out PyBaseObjectType, PyObject_GetAttrString(builtins, "object").StealNullable()); - SetPyMember(out PyNone, PyObject_GetAttrString(builtins, "None").StealNullable()); - SetPyMember(out PyTrue, PyObject_GetAttrString(builtins, "True").StealNullable()); - SetPyMember(out PyFalse, PyObject_GetAttrString(builtins, "False").StealNullable()); + SetPyMember(out _PyNone, PyObject_GetAttrString(builtins, "None").StealNullable()); + SetPyMember(out _PyTrue, PyObject_GetAttrString(builtins, "True").StealNullable()); + SetPyMember(out _PyFalse, PyObject_GetAttrString(builtins, "False").StealNullable()); - SetPyMemberTypeOf(out PyBoolType, PyTrue!); - SetPyMemberTypeOf(out PyNoneType, PyNone!); - SetPyMemberTypeOf(out PyTypeType, PyNoneType!); + SetPyMemberTypeOf(out PyBoolType, _PyTrue!); + SetPyMemberTypeOf(out PyNoneType, _PyNone!); SetPyMemberTypeOf(out PyMethodType, PyObject_GetAttrString(builtins, "len").StealNullable()); @@ -318,9 +312,9 @@ internal static void Shutdown(ShutdownMode mode) ClearClrModules(); RemoveClrRootModule(); - MoveClrInstancesOnwershipToPython(); - ClassManager.DisposePythonWrappersForClrTypes(); - TypeManager.RemoveTypes(); + NullGCHandles(ExtensionType.loadedExtensions); + ClassManager.RemoveClasses(); + TypeManager.RemoveTypes(mode); MetaType.Release(); PyCLRMetaType.Dispose(); @@ -332,12 +326,9 @@ internal static void Shutdown(ShutdownMode mode) DisposeLazyModule(inspect); PyObjectConversions.Reset(); - if (mode != ShutdownMode.Extension) - { - PyGC_Collect(); - bool everythingSeemsCollected = TryCollectingGarbage(); - Debug.Assert(everythingSeemsCollected); - } + PyGC_Collect(); + bool everythingSeemsCollected = TryCollectingGarbage(); + Debug.Assert(everythingSeemsCollected); Finalizer.Shutdown(); InternString.Shutdown(); @@ -388,9 +379,16 @@ static bool TryCollectingGarbage() GC.Collect(); GC.WaitForPendingFinalizers(); pyCollected += PyGC_Collect(); + pyCollected += Finalizer.Instance.DisposeAll(); } if (Volatile.Read(ref _collected) == 0 && pyCollected == 0) + { return true; + } + else + { + NullGCHandles(CLRObject.reflectedObjects); + } } return false; } @@ -529,36 +527,13 @@ private static void PyDictTryDelItem(BorrowedReference dict, string key) PyErr_Clear(); } - private static void MoveClrInstancesOnwershipToPython() + private static void NullGCHandles(IEnumerable objects) { - var objs = ManagedType.GetManagedObjects(); - var copyObjs = objs.ToArray(); - foreach (var entry in copyObjs) + foreach (IntPtr objWithGcHandle in objects.ToArray()) { - ManagedType obj = entry.Key; - if (!objs.ContainsKey(obj)) - { - System.Diagnostics.Debug.Assert(obj.gcHandle == default); - continue; - } - if (entry.Value == ManagedType.TrackTypes.Extension) - { - obj.CallTypeClear(); - // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), - // thus just be safe to give it back to GC chain. - if (!_PyObject_GC_IS_TRACKED(obj.ObjectReference)) - { - PyObject_GC_Track(obj.ObjectReference); - } - } - if (obj.gcHandle.IsAllocated) - { - obj.gcHandle.Free(); - ManagedType.SetGCHandle(obj.ObjectReference, default); - } - obj.gcHandle = default; + var @ref = new BorrowedReference(objWithGcHandle); + ManagedType.TryFreeGCHandle(@ref); } - ManagedType.ClearTrackedObjects(); } #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. @@ -579,7 +554,7 @@ private static void MoveClrInstancesOnwershipToPython() internal static PyObject PyFloatType; internal static PyType PyBoolType; internal static PyType PyNoneType; - internal static PyType PyTypeType; + internal static BorrowedReference PyTypeType => new(Delegates.PyType_Type); internal static int* Py_NoSiteFlag; @@ -594,9 +569,12 @@ private static void MoveClrInstancesOnwershipToPython() internal const int Py_GT = 4; internal const int Py_GE = 5; - internal static PyObject PyTrue; - internal static PyObject PyFalse; - internal static PyObject PyNone; + internal static BorrowedReference PyTrue => _PyTrue; + static PyObject _PyTrue; + internal static BorrowedReference PyFalse => _PyFalse; + static PyObject _PyFalse; + internal static BorrowedReference PyNone => _PyNone; + private static PyObject _PyNone; private static Lazy inspect; internal static PyObject InspectModule => inspect.Value; @@ -606,7 +584,7 @@ private static void MoveClrInstancesOnwershipToPython() internal static BorrowedReference CLRMetaType => PyCLRMetaType; - public static PyObject None => new(PyNone); + public static PyObject None => new(_PyNone); /// /// Check if any Python Exceptions occurred. @@ -640,7 +618,7 @@ internal static NewReference ExtendTuple(BorrowedReference t, params PyObject[] PyTuple_SetItem(items.Borrow(), size + n, args[n]); } - return items.AnalyzerWorkaround(); + return items; } internal static Type[]? PythonArgsToTypeArray(BorrowedReference arg) @@ -732,21 +710,11 @@ internal static unsafe void XIncref(BorrowedReference op) #endif } - -#if DEBUG - [Obsolete("Do not use")] -#else - [Obsolete("Do not use", error: true)] -#endif - internal static unsafe void XDecref(BorrowedReference op) - { - XDecref(StolenReference.DangerousFromPointer(op.DangerousGetAddress())); - } internal static unsafe void XDecref(StolenReference op) { #if DEBUG Debug.Assert(op == null || Refcount(new BorrowedReference(op.Pointer)) > 0); - Debug.Assert(_isInitialized || Py_IsInitialized() != 0); + Debug.Assert(_isInitialized || Py_IsInitialized() != 0 || _Py_IsFinalizing() != false); #endif #if !CUSTOM_INCDEC_REF if (op == null) return; @@ -1174,6 +1142,14 @@ internal static void _Py_NewReference(BorrowedReference ob) Delegates._Py_NewReference(ob); } + internal static bool? _Py_IsFinalizing() + { + if (Delegates._Py_IsFinalizing != null) + return Delegates._Py_IsFinalizing() != 0; + else + return null; ; + } + //==================================================================== // Python buffer API //==================================================================== @@ -1478,6 +1454,12 @@ internal static NewReference PyUnicode_InternFromString(string s) internal static int PyUnicode_Compare(BorrowedReference left, BorrowedReference right) => Delegates.PyUnicode_Compare(left, right); + internal static string ToString(BorrowedReference op) + { + using var strval = PyObject_Str(op); + return GetManagedStringFromUnicodeObject(strval.BorrowOrThrow())!; + } + /// /// Function to access the internal PyUnicode/PyString object and /// convert it to a managed string with the correct encoding. @@ -1503,7 +1485,7 @@ internal static NewReference PyUnicode_InternFromString(string s) return null; } - static string GetManagedStringFromUnicodeObject(in BorrowedReference op) + static string GetManagedStringFromUnicodeObject(BorrowedReference op) { #if DEBUG var type = PyObject_TYPE(op); @@ -1768,8 +1750,6 @@ internal static int PySys_SetObject(string name, BorrowedReference ob) internal static void PyType_Modified(BorrowedReference type) => Delegates.PyType_Modified(type); - internal static bool PyType_IsSubtype(BorrowedReference t1, IntPtr ofType) - => PyType_IsSubtype(t1, new BorrowedReference(ofType)); internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2) { Debug.Assert(t1 != null && t2 != null); @@ -1813,24 +1793,20 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); -#if DEBUG - [Obsolete("Do not use")] -#else - [Obsolete("Do not use", error: true)] -#endif - internal static void PyObject_GC_Del(BorrowedReference ob) - { - PyObject_GC_Del(StolenReference.DangerousFromPointer(ob.DangerousGetAddress())); - } - internal static void PyObject_GC_Del(StolenReference ob) => Delegates.PyObject_GC_Del(ob); - internal static void PyObject_GC_Track(BorrowedReference ob) => Delegates.PyObject_GC_Track(ob); + internal static bool PyObject_GC_IsTracked(BorrowedReference ob) + { + if (PyVersion >= new Version(3, 9)) + return Delegates.PyObject_GC_IsTracked(ob) != 0; + throw new NotSupportedException("Requires Python 3.9"); + } - internal static void PyObject_GC_UnTrack(BorrowedReference ob) => Delegates.PyObject_GC_UnTrack(ob); + internal static void PyObject_GC_Track(BorrowedReference ob) => Delegates.PyObject_GC_Track(ob); + internal static void PyObject_GC_UnTrack(BorrowedReference ob) => Delegates.PyObject_GC_UnTrack(ob); internal static void _PyObject_Dump(BorrowedReference ob) => Delegates._PyObject_Dump(ob); @@ -1924,41 +1900,6 @@ internal static int PyException_SetTraceback(BorrowedReference ex, BorrowedRefer internal static nint PyGC_Collect() => Delegates.PyGC_Collect(); - - internal static IntPtr _Py_AS_GC(BorrowedReference ob) - { - // XXX: PyGC_Head has a force alignment depend on platform. - // See PyGC_Head in objimpl.h for more details. - return ob.DangerousGetAddress() - (Is32Bit ? 16 : 24); - } - - internal static IntPtr _Py_FROM_GC(IntPtr gc) - { - return Is32Bit ? gc + 16 : gc + 24; - } - - internal static IntPtr _PyGCHead_REFS(IntPtr gc) - { - unsafe - { - var pGC = (PyGC_Head*)gc; - var refs = pGC->gc.gc_refs; - if (Is32Bit) - { - return new IntPtr(refs.ToInt32() >> _PyGC_REFS_SHIFT); - } - return new IntPtr(refs.ToInt64() >> _PyGC_REFS_SHIFT); - } - } - - internal static IntPtr _PyGC_REFS(BorrowedReference ob) - { - return _PyGCHead_REFS(_Py_AS_GC(ob)); - } - - internal static bool _PyObject_GC_IS_TRACKED(BorrowedReference ob) - => (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; - internal static void Py_CLEAR(BorrowedReference ob, int offset) => ReplaceReference(ob, offset, default); internal static void Py_CLEAR(ref T? ob) where T: PyObject @@ -2240,6 +2181,10 @@ static Delegates() PyObject_GenericGetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetDict), GetUnmanagedDll(PythonDLL)); PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); + try + { + PyObject_GC_IsTracked = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_IsTracked), GetUnmanagedDll(_PythonDll)); + } catch (MissingMethodException) { } PyObject_GC_Track = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Track), GetUnmanagedDll(_PythonDll)); PyObject_GC_UnTrack = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_UnTrack), GetUnmanagedDll(_PythonDll)); _PyObject_Dump = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_Dump), GetUnmanagedDll(_PythonDll)); @@ -2279,6 +2224,13 @@ static Delegates() _Py_NewReference = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_Py_NewReference), GetUnmanagedDll(_PythonDll)); } catch (MissingMethodException) { } + try + { + _Py_IsFinalizing = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_Py_IsFinalizing), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) { } + + PyType_Type = GetFunctionByName(nameof(PyType_Type), GetUnmanagedDll(_PythonDll)); } static global::System.IntPtr GetUnmanagedDll(string? libraryName) @@ -2497,6 +2449,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_GenericGetAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericSetAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Del { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GC_IsTracked { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Track { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_UnTrack { get; } internal static delegate* unmanaged[Cdecl] _PyObject_Dump { get; } @@ -2532,6 +2485,8 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyType_GetSlot { get; } internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; } internal static delegate* unmanaged[Cdecl] _Py_NewReference { get; } + internal static delegate* unmanaged[Cdecl] _Py_IsFinalizing { get; } + internal static IntPtr PyType_Type { get; } } } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 80e757453..4d49255e2 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -105,21 +105,12 @@ private static void RestoreRuntimeDataImpl() PyCLRMetaType = MetaType.RestoreRuntimeData(storage.Metatype); - var objs = RestoreRuntimeDataObjects(storage.SharedObjects); - // RestoreRuntimeDataModules(storage.Assmeblies); TypeManager.RestoreRuntimeData(storage.Types); - var clsObjs = ClassManager.RestoreRuntimeData(storage.Classes); - ImportHook.RestoreRuntimeData(storage.ImportHookState); + ClassManager.RestoreRuntimeData(storage.Classes); - foreach (var item in objs) - { - item.Value.ExecutePostActions(); - #warning XDecref(item.Key.pyHandle); - } - foreach (var item in clsObjs) - { - item.Value.ExecutePostActions(); - } + RestoreRuntimeDataObjects(storage.SharedObjects); + + ImportHook.RestoreRuntimeData(storage.ImportHookState); } public static bool HasStashData() @@ -147,69 +138,75 @@ static bool CheckSerializable (object o) private static SharedObjectsState SaveRuntimeDataObjects() { - var objs = ManagedType.GetManagedObjects(); - var extensionObjs = new List(); + var contexts = new Dictionary(PythonReferenceComparer.Instance); + var extensionObjs = new Dictionary(PythonReferenceComparer.Instance); + // make a copy with strongly typed references to avoid concurrent modification + var extensions = ExtensionType.loadedExtensions + .Select(addr => new PyObject( + new BorrowedReference(addr), + // if we don't skip collect, finalizer might modify loadedExtensions + skipCollect: true)) + .ToArray(); + foreach (var pyObj in extensions) + { + var extension = (ExtensionType)ManagedType.GetManagedObject(pyObj)!; + Debug.Assert(CheckSerializable(extension)); + var context = new InterDomainContext(); + contexts[pyObj] = context; + extension.Save(pyObj, context); + extensionObjs.Add(pyObj, extension); + } + var wrappers = new Dictionary>(); var userObjects = new CLRWrapperCollection(); - var contexts = new Dictionary(PythonReferenceComparer.Instance); - foreach (var entry in objs) + // make a copy with strongly typed references to avoid concurrent modification + var reflectedObjects = CLRObject.reflectedObjects + .Select(addr => new PyObject( + new BorrowedReference(addr), + // if we don't skip collect, finalizer might modify reflectedObjects + skipCollect: true)) + .ToList(); + foreach (var pyObj in reflectedObjects) { - var obj = entry.Key; - XIncref(obj.pyHandle); - switch (entry.Value) + // Wrapper must be the CLRObject + var clrObj = (CLRObject)ManagedType.GetManagedObject(pyObj)!; + object inst = clrObj.inst; + CLRMappedItem item; + List mappedObjs; + if (!userObjects.TryGetValue(inst, out item)) { - case ManagedType.TrackTypes.Extension: - Debug.Assert(CheckSerializable(obj)); - var context = new InterDomainContext(); - contexts[obj.pyHandle] = context; - obj.Save(context); - extensionObjs.Add(obj); - break; - case ManagedType.TrackTypes.Wrapper: - // Wrapper must be the CLRObject - var clrObj = (CLRObject)obj; - object inst = clrObj.inst; - CLRMappedItem item; - List mappedObjs; - if (!userObjects.TryGetValue(inst, out item)) - { - item = new CLRMappedItem(inst); - userObjects.Add(item); - - Debug.Assert(!wrappers.ContainsKey(inst)); - mappedObjs = new List(); - wrappers.Add(inst, mappedObjs); - } - else - { - mappedObjs = wrappers[inst]; - } - item.AddRef(clrObj.pyHandle); - mappedObjs.Add(clrObj); - break; - default: - break; + item = new CLRMappedItem(inst); + userObjects.Add(item); + + Debug.Assert(!wrappers.ContainsKey(inst)); + mappedObjs = new List(); + wrappers.Add(inst, mappedObjs); + } + else + { + mappedObjs = wrappers[inst]; } + item.AddRef(pyObj); + mappedObjs.Add(clrObj); } var wrapperStorage = new RuntimeDataStorage(); WrappersStorer?.Store(userObjects, wrapperStorage); - var internalStores = new List(); + var internalStores = new Dictionary(PythonReferenceComparer.Instance); foreach (var item in userObjects) { - if (!CheckSerializable(item.Instance)) + if (!item.Stored) { - continue; - } - internalStores.AddRange(wrappers[item.Instance]); - - foreach (var clrObj in wrappers[item.Instance]) - { - XIncref(clrObj.pyHandle); - var context = new InterDomainContext(); - contexts[clrObj.pyHandle] = context; - clrObj.Save(context); + if (!CheckSerializable(item.Instance)) + { + continue; + } + var clrO = wrappers[item.Instance].First(); + foreach (var @ref in item.PyRefs) + { + internalStores.Add(@ref, clrO); + } } } @@ -222,17 +219,18 @@ private static SharedObjectsState SaveRuntimeDataObjects() }; } - private static Dictionary RestoreRuntimeDataObjects(SharedObjectsState storage) + private static void RestoreRuntimeDataObjects(SharedObjectsState storage) { var extensions = storage.Extensions; var internalStores = storage.InternalStores; var contexts = storage.Contexts; - var storedObjs = new Dictionary(); - foreach (var obj in Enumerable.Union(extensions, internalStores)) + foreach (var extension in extensions) + { + extension.Value.Load(extension.Key, contexts[extension.Key]); + } + foreach (var clrObj in internalStores) { - var context = contexts[obj.pyHandle]; - obj.Load(context); - storedObjs.Add(obj, context); + clrObj.Value.Load(clrObj.Key, null); } if (WrappersStorer != null) { @@ -244,12 +242,10 @@ private static Dictionary RestoreRuntimeDataObj foreach (var pyRef in item.PyRefs ?? new List()) { var context = contexts[pyRef]; - var co = CLRObject.Restore(obj, pyRef, context); - storedObjs.Add(co, context); + CLRObject.Restore(obj, pyRef, context); } } } - return storedObjs; } private static IFormatter CreateFormatter() @@ -330,72 +326,5 @@ class InterDomainContext { private RuntimeDataStorage _storage; public RuntimeDataStorage Storage => _storage ?? (_storage = new RuntimeDataStorage()); - - /// - /// Actions after loaded. - /// - [NonSerialized] - private List _postActions; - public List PostActions => _postActions ?? (_postActions = new List()); - - public void AddPostAction(Action action) - { - PostActions.Add(action); - } - - public void ExecutePostActions() - { - if (_postActions == null) - { - return; - } - foreach (var action in _postActions) - { - action(); - } - } - } - - public class CLRMappedItem - { - public object Instance { get; private set; } - public List? PyRefs { get; set; } - - public CLRMappedItem(object instance) - { - Instance = instance; - } - - internal void AddRef(PyObject pyRef) - { - this.PyRefs ??= new List(); - this.PyRefs.Add(pyRef); - } - } - - - public interface ICLRObjectStorer - { - ICollection Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage); - CLRWrapperCollection Restore(RuntimeDataStorage storage); - } - - - public class CLRWrapperCollection : KeyedCollection - { - public bool TryGetValue(object key, out CLRMappedItem value) - { - if (Dictionary == null) - { - value = null; - return false; - } - return Dictionary.TryGetValue(key, out value); - } - - protected override object GetKeyForItem(CLRMappedItem item) - { - return item.Instance; - } } } diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index ac177d66f..3cd842d39 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Runtime.InteropServices; using static Python.Runtime.Runtime; @@ -9,9 +8,6 @@ namespace Python.Runtime { class RuntimeState { - public static bool ShouldRestoreObjects { get; set; } = false; - public static bool UseDummyGC { get; set; } = false; - public static void Save() { if (!PySys_GetObject("dummy_gc").IsNull) @@ -19,72 +15,23 @@ public static void Save() throw new Exception("Runtime State set already"); } - NewReference objs = default; - if (ShouldRestoreObjects) - { - objs = PySet_New(default); - foreach (var objRaw in PyGCGetObjects()) - { - AddObjPtrToSet(objs.Borrow(), new BorrowedReference(objRaw)); - } - } - using var modules = PySet_New(default); - foreach (var name in GetModuleNames()) - { - int res = PySet_Add(modules.Borrow(), new BorrowedReference(name)); - PythonException.ThrowIfIsNotZero(res); - } - + int res = PySys_SetObject("initial_modules", modules.Borrow()); + PythonException.ThrowIfIsNotZero(res); - var dummyGCHead = PyMem_Malloc(Marshal.SizeOf(typeof(PyGC_Head))); - unsafe - { - var head = (PyGC_Head*)dummyGCHead; - head->gc.gc_next = dummyGCHead; - head->gc.gc_prev = dummyGCHead; - head->gc.gc_refs = IntPtr.Zero; - } + foreach (var name in GetModuleNames()) { - using var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); - int res = PySys_SetObject("dummy_gc", pyDummyGC.Borrow()); - PythonException.ThrowIfIsNotZero(res); - - res = PySys_SetObject("initial_modules", modules.Borrow()); + res = PySet_Add(modules.Borrow(), new BorrowedReference(name)); PythonException.ThrowIfIsNotZero(res); - - if (ShouldRestoreObjects) - { - AddObjPtrToSet(objs.Borrow(), modules.Borrow()); - try - { - res = PySys_SetObject("initial_objs", objs.Borrow()); - PythonException.ThrowIfIsNotZero(res); - } - finally - { - objs.Dispose(); - } - } } } public static void Restore() { - var dummyGCAddr = PySys_GetObject("dummy_gc"); - if (dummyGCAddr.IsNull) - { - throw new InvalidOperationException("Runtime state have not set"); - } - var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); - ResotreModules(dummyGC); - if (ShouldRestoreObjects) - { - RestoreObjects(dummyGC); - } + ResotreModules(); } - private static void ResotreModules(IntPtr dummyGC) + private static void ResotreModules() { var intialModules = PySys_GetObject("initial_modules"); Debug.Assert(!intialModules.IsNull); @@ -96,12 +43,6 @@ private static void ResotreModules(IntPtr dummyGC) { continue; } - var module = PyDict_GetItem(modules, name); - - if (UseDummyGC && _PyObject_GC_IS_TRACKED(module)) - { - ExchangeGCChain(module, dummyGC); - } if (PyDict_DelItem(modules, name) != 0) { PyErr_Print(); @@ -109,41 +50,6 @@ private static void ResotreModules(IntPtr dummyGC) } } - private static void RestoreObjects(IntPtr dummyGC) - { - if (!UseDummyGC) - { - throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); - } - BorrowedReference intialObjs = PySys_GetObject("initial_objs"); - Debug.Assert(@intialObjs.IsNull); - foreach (var objRaw in PyGCGetObjects()) - { - using var p = PyLong_FromVoidPtr(objRaw); - var obj = new BorrowedReference(objRaw); - if (PySet_Contains(intialObjs, p.Borrow()) == 1) - { - continue; - } - Debug.Assert(_PyObject_GC_IS_TRACKED(obj), "A GC object must be tracked"); - ExchangeGCChain(obj, dummyGC); - } - } - - public static IEnumerable PyGCGetObjects() - { - using var gc = PyModule.Import("gc"); - using var get_objects = gc.GetAttr("get_objects"); - using var objs = new PyObject(PyObject_CallObject(get_objects, args: null).StealOrThrow()); - nint length = PyList_Size(objs); - if (length < 0) throw PythonException.ThrowLastAsClrException(); - for (nint i = 0; i < length; i++) - { - BorrowedReference obj = PyList_GetItem(objs, i); - yield return obj.DangerousGetAddress(); - } - } - public static IEnumerable GetModuleNames() { var modules = PyImport_GetModuleDict(); @@ -157,62 +63,5 @@ public static IEnumerable GetModuleNames() } return result; } - - private static void AddObjPtrToSet(BorrowedReference set, BorrowedReference obj) - { - IntPtr objRaw = obj.DangerousGetAddress(); - using var p = PyLong_FromVoidPtr(objRaw); - XIncref(obj); - int res = PySet_Add(set, p.Borrow()); - PythonException.ThrowIfIsNotZero(res); - } - /// - /// Exchange gc to a dummy gc prevent nullptr error in _PyObject_GC_UnTrack macro. - /// - private static void ExchangeGCChain(BorrowedReference obj, IntPtr gc) - { - var head = _Py_AS_GC(obj); - if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) - { - throw new ArgumentException("GC object untracked"); - } - unsafe - { - var g = (PyGC_Head*)head; - var newGCGen = (PyGC_Head*)gc; - - ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = g->gc.gc_next; - ((PyGC_Head*)g->gc.gc_next)->gc.gc_prev = g->gc.gc_prev; - - g->gc.gc_next = gc; - g->gc.gc_prev = newGCGen->gc.gc_prev; - ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = head; - newGCGen->gc.gc_prev = head; - } - } - - private static IEnumerable IterGCNodes(IntPtr gc) - { - var node = GetNextGCNode(gc); - while (node != gc) - { - var next = GetNextGCNode(node); - yield return node; - node = next; - } - } - - private static IEnumerable IterObjects(IntPtr gc) - { - foreach (var node in IterGCNodes(gc)) - { - yield return _Py_FROM_GC(node); - } - } - - private static unsafe IntPtr GetNextGCNode(IntPtr node) - { - return ((PyGC_Head*)node)->gc.gc_next; - } } } diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs index 1f732b8be..669285fe1 100644 --- a/src/runtime/slots/mp_length.cs +++ b/src/runtime/slots/mp_length.cs @@ -9,23 +9,6 @@ namespace Python.Runtime.Slots { internal static class mp_length_slot { - private static MethodInfo? _lengthMethod; - public static MethodInfo Method - { - get - { - if (_lengthMethod != null) - { - return _lengthMethod; - } - _lengthMethod = typeof(mp_length_slot).GetMethod( - nameof(mp_length_slot.mp_length), - BindingFlags.Static | BindingFlags.NonPublic); - Debug.Assert(_lengthMethod != null); - return _lengthMethod!; - } - } - public static bool CanAssign(Type clrType) { if (typeof(ICollection).IsAssignableFrom(clrType)) @@ -47,7 +30,7 @@ public static bool CanAssign(Type clrType) /// Implements __len__ for classes that implement ICollection /// (this includes any IList implementer or Array subclass) /// - private static nint mp_length(BorrowedReference ob) + internal static nint impl(BorrowedReference ob) { var co = ManagedType.GetManagedObject(ob) as CLRObject; if (co == null) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 4b1425e25..834703e80 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,13 +1,12 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using System.Diagnostics; -using Python.Runtime.Slots; +using Python.Runtime.Native; using Python.Runtime.StateSerialization; -using static Python.Runtime.PythonException; + namespace Python.Runtime { @@ -25,8 +24,7 @@ internal class TypeManager private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; private static Dictionary cache = new(); - private static readonly Dictionary _slotsHolders = new Dictionary(PythonReferenceComparer.Instance); - private static Dictionary _slotsImpls = new Dictionary(); + static readonly Dictionary _slotsHolders = new Dictionary(PythonReferenceComparer.Instance); // Slots which must be set private static readonly string[] _requiredSlots = new string[] @@ -47,12 +45,12 @@ internal static void Initialize() pythonBaseTypeProvider = PythonEngine.InteropConfiguration.pythonBaseTypeProviders; } - internal static void RemoveTypes() + internal static void RemoveTypes(ShutdownMode shutdownMode) { foreach (var type in cache.Values) { - SlotsHolder holder; - if (_slotsHolders.TryGetValue(type, out holder)) + if (shutdownMode == ShutdownMode.Extension + && _slotsHolders.TryGetValue(type, out var holder)) { // If refcount > 1, it needs to reset the managed slot, // otherwise it can dealloc without any trick. @@ -64,7 +62,6 @@ internal static void RemoveTypes() type.Dispose(); } cache.Clear(); - _slotsImpls.Clear(); _slotsHolders.Clear(); } @@ -72,26 +69,19 @@ internal static TypeManagerState SaveRuntimeData() => new() { Cache = cache, - SlotImplementations = _slotsImpls, }; internal static void RestoreRuntimeData(TypeManagerState storage) { Debug.Assert(cache == null || cache.Count == 0); - _slotsImpls = storage.SlotImplementations; var typeCache = storage.Cache; foreach (var entry in typeCache) { - if (!entry.Key.Valid) - { - entry.Value.Dispose(); - continue; - } Type type = entry.Key.Value;; cache[type] = entry.Value; - SlotsHolder holder = CreateSolotsHolder(entry.Value); - InitializeSlots(entry.Value, _slotsImpls[type], holder); - // FIXME: mp_length_slot.CanAssgin(clrType) + SlotsHolder holder = CreateSlotsHolder(entry.Value); + InitializeSlots(entry.Value, type, holder); + Runtime.PyType_Modified(entry.Value); } } @@ -103,7 +93,6 @@ internal static PyType GetType(Type type) { pyType = CreateType(type); cache[type] = pyType; - _slotsImpls.Add(type, type); } return pyType; } @@ -115,52 +104,6 @@ internal static PyType GetType(Type type) /// internal static BorrowedReference GetTypeReference(Type type) => GetType(type).Reference; - - /// - /// Get the fully initialized Python type that reflects the given CLR type. - /// The given ManagedType instance is a managed object that implements - /// the appropriate semantics in Python for the reflected managed type. - /// - internal static PyType GetOrInitializeClass(ClassBase obj, Type type) - { - var pyType = GetOrCreateClass(type); - if (!pyType.IsReady) - { - InitializeClass(pyType, obj, type); - _slotsImpls.Add(type, obj.GetType()); - } - return pyType; - } - - /// - /// Get the Python type that reflects the given CLR type. - /// The given ManagedType instance is a managed object that implements - /// the appropriate semantics in Python for the reflected managed type. - /// - /// - /// Returned might be partially initialized. - /// If you need fully initialized type, use - /// - internal static PyType GetOrCreateClass(Type type) - { - if (!cache.TryGetValue(type, out var pyType)) - { - pyType = AllocateClass(type); - cache.Add(type, pyType); - try - { - InitializeClass(type, pyType); - } - catch - { - cache.Remove(type); - throw; - } - } - return pyType; - } - - /// /// The following CreateType implementations do the necessary work to /// create Python types to represent managed extension types, reflected @@ -178,6 +121,8 @@ internal static unsafe PyType CreateType(Type impl) ? Runtime.PyModuleType : Runtime.PyBaseObjectType; + type.BaseReference = base_; + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); int tp_clr_inst_offset = newFieldOffset; @@ -189,7 +134,7 @@ internal static unsafe PyType CreateType(Type impl) Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, tp_clr_inst_offset); Util.WriteIntPtr(type, TypeOffset.tp_new, (IntPtr)Runtime.Delegates.PyType_GenericNew); - SlotsHolder slotsHolder = CreateSolotsHolder(type); + SlotsHolder slotsHolder = CreateSlotsHolder(type); InitializeSlots(type, impl, slotsHolder); type.Flags = TypeFlags.Default | TypeFlags.HasClrInstance | @@ -214,13 +159,17 @@ internal static unsafe PyType CreateType(Type impl) } - static void InitializeClass(Type clrType, PyType pyType) + internal static void InitializeClassCore(Type clrType, PyType pyType, ClassBase impl) { if (pyType.BaseReference != null) { return; } + // Hide the gchandle of the implementation in a magic type slot. + GCHandle gc = GCHandle.Alloc(impl); + ManagedType.InitGCHandle(pyType, Runtime.CLRMetaType, gc); + using var baseTuple = GetBaseTypeTuple(clrType); InitializeBases(pyType, baseTuple); @@ -229,21 +178,7 @@ static void InitializeClass(Type clrType, PyType pyType) InitializeCoreFields(pyType); } - static PyType AllocateClass(Type clrType) - { - string name = GetPythonTypeName(clrType); - - var type = AllocateTypeObject(name, Runtime.PyCLRMetaType); - type.Flags = TypeFlags.Default - | TypeFlags.HasClrInstance - | TypeFlags.HeapType - | TypeFlags.BaseType - | TypeFlags.HaveGC; - - return type; - } - - static string GetPythonTypeName(Type clrType) + internal static string GetPythonTypeName(Type clrType) { var result = new System.Text.StringBuilder(); GetPythonTypeName(clrType, target: result); @@ -336,45 +271,20 @@ static void InitializeCoreFields(PyType type) Util.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); } - static void InitializeClass(PyType type, ClassBase impl, Type clrType) + internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) { // we want to do this after the slot stuff above in case the class itself implements a slot method - SlotsHolder slotsHolder = CreateSolotsHolder(type); + SlotsHolder slotsHolder = CreateSlotsHolder(type); InitializeSlots(type, impl.GetType(), slotsHolder); - if (Util.ReadIntPtr(type, TypeOffset.mp_length) == IntPtr.Zero - && mp_length_slot.CanAssign(clrType)) - { - InitializeSlot(type, TypeOffset.mp_length, mp_length_slot.Method, slotsHolder); - } - - if (!typeof(IEnumerable).IsAssignableFrom(clrType) && - !typeof(IEnumerator).IsAssignableFrom(clrType)) - { - // The tp_iter slot should only be set for enumerable types. - Util.WriteIntPtr(type, TypeOffset.tp_iter, IntPtr.Zero); - } - - - // Only set mp_subscript and mp_ass_subscript for types with indexers - if (!(impl is ArrayObject)) - { - if (impl.indexer == null || !impl.indexer.CanGet) - { - Util.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); - } - if (impl.indexer == null || !impl.indexer.CanSet) - { - Util.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); - } - } + impl.InitializeSlots(type, slotsHolder); OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. - if (Runtime.PyType_Ready(type) != 0) + if (!type.IsReady && Runtime.PyType_Ready(type) != 0) { throw PythonException.ThrowLastAsClrException(); } @@ -384,16 +294,6 @@ static void InitializeClass(PyType type, ClassBase impl, Type clrType) using (var mod = Runtime.PyString_FromString(mn)) Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod.Borrow()); - // Hide the gchandle of the implementation in a magic type slot. - GCHandle gc = impl.AllocGCHandle(); - ManagedType.InitGCHandle(type, Runtime.CLRMetaType, gc); - - // Set the handle attributes on the implementing instance. - impl.tpHandle = type; - impl.pyHandle = type; - - impl.InitializeSlots(slotsHolder); - Runtime.PyType_Modified(type.Reference); //DebugUtil.DumpType(type); @@ -454,12 +354,17 @@ internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedRe { // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation - string name = Runtime.GetManagedString(py_name); + string? name = Runtime.GetManagedString(py_name); + if (name is null) + { + Exceptions.SetError(Exceptions.ValueError, "Class name must not be None"); + return default; + } // the derived class can have class attributes __assembly__ and __module__ which // control the name of the assembly and module the new type is created in. - object assembly = null; - object namespaceStr = null; + object? assembly = null; + object? namespaceStr = null; using (var assemblyKey = new PyString("__assembly__")) { @@ -494,48 +399,22 @@ internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedRe return Exceptions.RaiseTypeError("invalid base class, expected CLR class type"); } - try - { - Type subType = ClassDerivedObject.CreateDerivedType(name, - baseClass.type.Value, - dictRef, - (string)namespaceStr, - (string)assembly); - - // create the new ManagedType and python type - ClassBase subClass = ClassManager.GetClass(subType); - var py_type = GetOrInitializeClass(subClass, subType); - - // by default the class dict will have all the C# methods in it, but as this is a - // derived class we want the python overrides in there instead if they exist. - var cls_dict = Util.ReadRef(py_type, TypeOffset.tp_dict); - ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, dictRef)); - // Update the __classcell__ if it exists - BorrowedReference cell = Runtime.PyDict_GetItemString(cls_dict, "__classcell__"); - if (!cell.IsNull) - { - ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type)); - ThrowIfIsNotZero(Runtime.PyDict_DelItemString(cls_dict, "__classcell__")); - } - - return new NewReference(py_type); - } - catch (Exception e) - { - return Exceptions.RaiseTypeError(e.Message); - } + return ReflectedClrType.CreateSubclass(baseClass, name, + ns: (string?)namespaceStr, + assembly: (string?)assembly, + dict: dictRef); } - internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, int flags, IntPtr doc) + internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) { Marshal.WriteIntPtr(mdef, name); Marshal.WriteIntPtr(mdef, 1 * IntPtr.Size, func); - Marshal.WriteInt32(mdef, 2 * IntPtr.Size, flags); + Marshal.WriteInt32(mdef, 2 * IntPtr.Size, (int)flags); Marshal.WriteIntPtr(mdef, 3 * IntPtr.Size, doc); return mdef + 4 * IntPtr.Size; } - internal static IntPtr WriteMethodDef(IntPtr mdef, string name, IntPtr func, int flags = 0x0001, + internal static IntPtr WriteMethodDef(IntPtr mdef, string name, IntPtr func, PyMethodFlags flags = PyMethodFlags.VarArgs, string? doc = null) { IntPtr namePtr = Marshal.StringToHGlobalAnsi(name); @@ -567,6 +446,27 @@ internal static void FreeMethodDef(IntPtr mdef) } } + internal static PyType CreateMetatypeWithGCHandleOffset() + { + var py_type = new PyType(Runtime.PyTypeType, prevalidated: true); + int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst_offset + ; + var result = new PyType(new TypeSpec("GC Offset Base", basicSize: size, + new TypeSpec.Slot[] + { + + }, + TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC), + bases: new PyTuple(new[] { py_type })); + + SetRequiredSlots(result, seen: new HashSet()); + + Runtime.PyType_Modified(result); + + return result; + } + internal static PyType CreateMetaType(Type impl, out SlotsHolder slotsHolder) { // The managed metatype is functionally little different than the @@ -574,21 +474,22 @@ internal static PyType CreateMetaType(Type impl, out SlotsHolder slotsHolder) // the standard type slots, and has to subclass PyType_Type for // certain functions in the C runtime to work correctly with it. - PyType type = AllocateTypeObject("CLR Metatype", metatype: Runtime.PyTypeType); + PyType gcOffsetBase = CreateMetatypeWithGCHandleOffset(); - PyType py_type = Runtime.PyTypeType; - Util.WriteRef(type, TypeOffset.tp_base, new NewReference(py_type).Steal()); + PyType type = AllocateTypeObject("CLR Metatype", metatype: gcOffsetBase); - int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) - + IntPtr.Size // tp_clr_inst_offset + Util.WriteRef(type, TypeOffset.tp_base, new NewReference(gcOffsetBase).Steal()); + + nint size = Util.ReadInt32(gcOffsetBase, TypeOffset.tp_basicsize) + IntPtr.Size // tp_clr_inst ; - Util.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); + Util.WriteIntPtr(type, TypeOffset.tp_basicsize, size); Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, ManagedType.Offsets.tp_clr_inst); const TypeFlags flags = TypeFlags.Default | TypeFlags.HeapType - | TypeFlags.HaveGC; + | TypeFlags.HaveGC + | TypeFlags.HasClrInstance; Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); // Slots will inherit from TypeType, it's not neccesary for setting them. @@ -602,7 +503,7 @@ internal static PyType CreateMetaType(Type impl, out SlotsHolder slotsHolder) { throw PythonException.ThrowLastAsClrException(); } - + BorrowedReference dict = Util.ReadRef(type, TypeOffset.tp_dict); using (var mod = Runtime.PyString_FromString("CLR")) Runtime.PyDict_SetItemString(dict, "__module__", mod.Borrow()); @@ -758,6 +659,11 @@ internal static void InitializeSlots(PyType type, Type impl, SlotsHolder slotsHo impl = impl.BaseType; } + SetRequiredSlots(type, seen); + } + + private static void SetRequiredSlots(PyType type, HashSet seen) + { foreach (string slot in _requiredSlots) { if (seen.Contains(slot)) @@ -791,6 +697,12 @@ internal static void InitializeSlot(BorrowedReference type, int slotOffset, Dele InitializeSlot(type, slotOffset, thunk, slotsHolder); } + internal static void InitializeSlotIfEmpty(BorrowedReference type, int slotOffset, Delegate impl, SlotsHolder slotsHolder) + { + if (slotsHolder.IsHolding(slotOffset)) return; + InitializeSlot(type, slotOffset, impl, slotsHolder); + } + static void InitializeSlot(BorrowedReference type, int slotOffset, ThunkInfo thunk, SlotsHolder slotsHolder) { Util.WriteIntPtr(type, slotOffset, thunk.Address); @@ -800,7 +712,6 @@ static void InitializeSlot(BorrowedReference type, int slotOffset, ThunkInfo thu } } - /// /// /// Utility method to copy slots from a given type to another type. /// @@ -810,15 +721,12 @@ internal static void CopySlot(BorrowedReference from, BorrowedReference to, int Util.WriteIntPtr(to, offset, fp); } - private static SlotsHolder CreateSolotsHolder(PyType type) + internal static SlotsHolder CreateSlotsHolder(PyType type) { var holder = new SlotsHolder(type); _slotsHolders.Add(type, holder); return holder; } - - internal static SlotsHolder GetSlotsHolder(PyType type) - => _slotsHolders[type]; } @@ -834,6 +742,8 @@ class SlotsHolder private readonly PyType Type; + public string?[] Holds => _slots.Keys.Select(TypeOffset.GetSlotName).ToArray(); + /// /// Create slots holder for holding the delegate of slots and be able to reset them. /// @@ -845,6 +755,8 @@ public SlotsHolder(PyType type) public bool IsHolding(int offset) => _slots.ContainsKey(offset); + public ICollection Slots => _slots.Keys; + public void Set(int offset, ThunkInfo thunk) { _slots[offset] = thunk; @@ -865,6 +777,18 @@ public void KeeapAlive(ThunkInfo thunk) _keepalive.Add(thunk); } + public static void ResetSlots(BorrowedReference type, IEnumerable slots) + { + foreach (int offset in slots) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Util.WriteIntPtr(type, offset, ptr); + } + } + public void ResetSlots() { if (_alreadyReset) @@ -876,14 +800,7 @@ public void ResetSlots() IntPtr tp_name = Util.ReadIntPtr(Type, TypeOffset.tp_name); string typeName = Marshal.PtrToStringAnsi(tp_name); #endif - foreach (var offset in _slots.Keys) - { - IntPtr ptr = GetDefaultSlot(offset); -#if DEBUG - //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); -#endif - Util.WriteIntPtr(Type, offset, ptr); - } + ResetSlots(Type, _slots.Keys); foreach (var action in _deallocators) { @@ -906,15 +823,7 @@ public void ResetSlots() if (Type != Runtime.CLRMetaType) { var metatype = Runtime.PyObject_TYPE(Type); - if (ManagedType.TryGetGCHandle(Type, metatype) is { } handle) - { - if (handle.IsAllocated) - { - handle.Free(); - } - - ManagedType.SetGCHandle(Type, metatype, default); - } + ManagedType.TryFreeGCHandle(Type, metatype); } } diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index 66fb4f894..cec380467 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -310,7 +310,7 @@ public class Cls public static event Action Before; public static void Call() { - Before(); + if (Before != null) Before(); } } }", @@ -324,7 +324,7 @@ public class Cls public static event Action After; public static void Call() { - After(); + if (After != null) After(); } } }", @@ -335,21 +335,29 @@ import sys from TestNamespace import Cls called = False +before_reload_called = False +after_reload_called = False def callback_function(): global called called = True def before_reload(): - global called + global called, before_reload_called called = False Cls.Before += callback_function Cls.Call() assert called is True + before_reload_called = True def after_reload(): - global called - assert called is True + global called, after_reload_called, before_reload_called + + assert before_reload_called is True + if not after_reload_called: + assert called is True + after_reload_called = True + called = False Cls.Call() assert called is False @@ -762,12 +770,12 @@ def before_reload(): sys.my_cls = TestNamespace.Before def after_reload(): - bar = sys.my_cls() - - # Don't crash! - print(bar) - print(bar.__str__()) - print(bar.__repr__()) + try: + bar = sys.my_cls() + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') ", }, @@ -1159,7 +1167,7 @@ public static int Main() }} catch (Exception e) {{ - Console.WriteLine(e.StackTrace); + Console.Error.WriteLine(e.StackTrace); throw; }} return 0; @@ -1173,18 +1181,27 @@ public static int Main() public static int Main(string[] args) { - TestCase testCase; if (args.Length < 1) { - testCase = Cases[0]; + foreach (var testCase in Cases) + { + Run(testCase); + Console.WriteLine(); + } } else { string testName = args[0]; Console.WriteLine($"-- Looking for domain reload test case {testName}"); - testCase = Cases.First(c => c.Name == testName); + var testCase = int.TryParse(testName, out var index) ? Cases[index] : Cases.First(c => c.Name == testName); + Run(testCase); } + return 0; + } + + static void Run(TestCase testCase) + { Console.WriteLine($"-- Running domain reload test case: {testCase.Name}"); SetupTestFolder(testCase.Name); @@ -1222,7 +1239,7 @@ public static int Main(string[] args) // folder behind to debug failing tests. TeardownTestFolder(); - return 0; + Console.WriteLine($"-- PASSED: {testCase.Name}"); } static void SetupTestFolder(string testCaseName) diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index e7a82ded2..f0890c7c3 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -56,7 +56,6 @@ def test_property_visibility_change(): def test_class_visibility_change(): _run_test("class_visibility_change") -@pytest.mark.skip(reason='FIXME: Domain reload fails when Python points to a .NET object which points back to Python objects') def test_method_parameters_change(): _run_test("method_parameters_change") @@ -70,7 +69,6 @@ def test_field_type_change(): def test_rename_event(): _run_test('event_rename') -@pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") def test_construct_removed_class(): _run_test("construct_removed_class") @@ -90,4 +88,4 @@ def test_nested_type(): _run_test("nested_type") def test_import_after_reload(): - _run_test("import_after_reload") \ No newline at end of file + _run_test("import_after_reload")