From 7736001550dad09fdf9857a7948dbaf25ef7b0bd Mon Sep 17 00:00:00 2001 From: David Lassonde Date: Wed, 4 Jul 2018 15:45:48 -0400 Subject: [PATCH 01/10] * Now calling PythonEngine.Shutdown on app domain unload --- src/runtime/pythonengine.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index a23c7ac79..d8c819308 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -145,6 +145,19 @@ public static void Initialize(bool setSysArgv = true) Initialize(Enumerable.Empty(), setSysArgv: setSysArgv); } + /// + /// On Domain Unload Event Handler + /// + /// + /// Performs necessary tasks (shutdown) when the current app domain + /// gets unloaded, leaving the engine, the runtime and the Python + /// interpreter in consistent states + /// + private static void OnDomainUnload(object sender, EventArgs e) + { + Shutdown(); + } + /// /// Initialize Method /// @@ -158,6 +171,9 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) { if (!initialized) { + // Make sure we shut down properly on app domain reload + System.AppDomain.CurrentDomain.DomainUnload += new EventHandler(OnDomainUnload); + // Creating the delegateManager MUST happen before Runtime.Initialize // is called. If it happens afterwards, DelegateManager's CodeGenerator // throws an exception in its ctor. This exception is eaten somehow From c0729f3335cc5d849003cf20f12739f2439464a7 Mon Sep 17 00:00:00 2001 From: David Lassonde Date: Thu, 5 Jul 2018 15:37:20 -0400 Subject: [PATCH 02/10] * Unsubscribing the app domain reload event handler on shutdown --- src/runtime/pythonengine.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index d8c819308..f05d641ed 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -310,6 +310,9 @@ public static void Shutdown() { if (initialized) { + // Make sure we shut down properly on app domain reload + System.AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + PyScopeManager.Global.Clear(); Marshal.FreeHGlobal(_pythonHome); _pythonHome = IntPtr.Zero; From 1b38936ed1dc38ff86115a04480aac32e6800e16 Mon Sep 17 00:00:00 2001 From: vkovec Date: Tue, 7 Aug 2018 14:31:40 -0400 Subject: [PATCH 03/10] refactor platform #defines to be if statements - so that the platform is decided at runtime instead of compile time --- src/runtime/runtime.cs | 215 +++++++++++++++++++++++++++++++---------- 1 file changed, 163 insertions(+), 52 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b08a56622..535e06d3b 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -5,55 +5,114 @@ namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods + internal static class OSType { -#if MONO_LINUX || MONO_OSX -#if NETSTANDARD - private static int RTLD_NOW = 0x2; -#if MONO_LINUX - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; - public static IntPtr LoadLibrary(string fileName) + // TODO: find how to differentiate between linux and osx + public static bool IsLinux { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); + get + { + return Environment.OSVersion.Platform == PlatformID.Unix; + // return System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Linux); + } } -#elif MONO_OSX - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib" - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - public static IntPtr LoadLibrary(string fileName) + public static bool IsWindows + { + get + { + var os = Environment.OSVersion.Platform; + return os != PlatformID.MacOSX && os != PlatformID.Unix; + } + } + + public static bool IsOSX { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); + get + { + return Environment.OSVersion.Platform == PlatformID.MacOSX || Environment.OSVersion.Platform == PlatformID.Unix; + } } -#endif + } + + [SuppressUnmanagedCodeSecurity] + internal static class NativeMethods + { + private static IntPtr RTLD_DEFAULT + { + get + { + if (OSType.IsOSX) + { + return new IntPtr(-2); + } + return IntPtr.Zero; + } + } + + private const string linuxNativeDll = "libdl.so"; + +#if NETSTANDARD + private const string osxNativeDLL = "/usr/lib/libSystem.dylib"; #else + private const string osxNativeDll = "__Internal"; +#endif + private static int RTLD_NOW = 0x2; + + private static int RTLD_GLOBAL + { + get + { + if (OSType.IsOSX) + { + return 0x8; + } + return 0x100; + } + } + private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif public static IntPtr LoadLibrary(string fileName) { + if (OSType.IsWindows) + { + return LoadLibrary_win(fileName); + } + +#if NETSTANDARD + if (IsOSX) + { + string file = $"lib{fileName}.dylib"; + } + else + { + string file = $"lib{fileName}.so"; + } + return dlopen(file, RTLD_NOW | RTLD_GLOBAL); +#else return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } #endif - + } public static void FreeLibrary(IntPtr handle) { + if (OSType.IsWindows) + { + FreeLibrary_win(handle); + return; + } dlclose(handle); } public static IntPtr GetProcAddress(IntPtr dllHandle, string name) { + if (OSType.IsWindows) + { + return GetProcAddress_win(dllHandle, name); + } + // look in the exe if dllHandle is NULL if (dllHandle == IntPtr.Zero) { @@ -70,30 +129,69 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) } return res; } + + public static IntPtr dlopen(String fileName, int flags) + { + if (OSType.IsOSX) { return dlopen_mac(fileName, flags); } + return dlopen_linux(fileName, flags); + } + + private static IntPtr dlsym(IntPtr handle, String symbol) + { + if (OSType.IsOSX) { return dlsym_mac(handle, symbol); } + return dlsym_linux(handle, symbol); + } + + private static int dlclose(IntPtr handle) + { + if (OSType.IsOSX) { return dlclose_mac(handle); } + return dlclose_linux(handle); + } + + private static IntPtr dlerror() + { + if (OSType.IsOSX) { return dlerror_mac(); } + return dlerror_linux(); + } - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); + // ------------- Linux ---------------- + [DllImport(linuxNativeDll, EntryPoint = "dlopen", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen_linux(String fileName, int flags); - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); + [DllImport(linuxNativeDll, EntryPoint = "dlsym", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym_linux(IntPtr handle, String symbol); - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); + [DllImport(linuxNativeDll, EntryPoint = "dlclose", CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose_linux(IntPtr handle); - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; + [DllImport(linuxNativeDll, EntryPoint = "dlerror", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror_linux(); - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); + // ------------- Mac ----------------- + [DllImport(osxNativeDll, EntryPoint = "dlopen", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen_mac(String fileName, int flags); - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + [DllImport(osxNativeDll, EntryPoint = "dlsym", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym_mac(IntPtr handle, String symbol); - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif + [DllImport(osxNativeDll, EntryPoint = "dlclose", CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose_mac(IntPtr handle); + + [DllImport(osxNativeDll, EntryPoint = "dlerror", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror_mac(); + + //#else // Windows + private const string winNativeDll = "kernel32.dll"; + + [DllImport(winNativeDll, EntryPoint = "LoadLibrary")] + public static extern IntPtr LoadLibrary_win(string dllToLoad); + + [DllImport(winNativeDll, EntryPoint = "GetProcAddress")] + public static extern IntPtr GetProcAddress_win(IntPtr hModule, string procedureName); + + [DllImport(winNativeDll, EntryPoint = "FreeLibrary")] + public static extern bool FreeLibrary_win(IntPtr hModule); +//#endif } /// @@ -155,11 +253,22 @@ public class Runtime #error You must define one of PYTHON34 to PYTHON37 or PYTHON27 #endif -#if MONO_LINUX || MONO_OSX // Linux/macOS use dotted version string - internal const string dllBase = "python" + _pyversion; -#else // Windows + // TODO : ideally find a way to do this without having to rename the dylib on Mac/Linux + /* private static string _dllPostFix + { + get + { + if (OSType.IsWindows) + { + return _pyver; + } + // Linux/macOS use dotted version string + return _pyversion; + } + } + internal const string dllBase = "python" + _dllPostFix;*/ internal const string dllBase = "python" + _pyver; -#endif + #if PYTHON_WITH_PYDEBUG internal const string dllWithPyDebug = "d"; @@ -325,12 +434,14 @@ internal static void Initialize() } _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); -#if !(MONO_LINUX || MONO_OSX) - if (dllLocal != IntPtr.Zero) + //#if !(MONO_LINUX || MONO_OSX) + if (!(OSType.IsLinux || OSType.IsOSX)) { - NativeMethods.FreeLibrary(dllLocal); + if (dllLocal != IntPtr.Zero) + { + NativeMethods.FreeLibrary(dllLocal); + } } -#endif // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); From 36e973786f4a85f1c95203ef6a5b3ad430526d38 Mon Sep 17 00:00:00 2001 From: vkovec Date: Tue, 7 Aug 2018 14:32:23 -0400 Subject: [PATCH 04/10] remove commented out code --- src/runtime/runtime.cs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 535e06d3b..cd31607b3 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -13,7 +13,6 @@ public static bool IsLinux get { return Environment.OSVersion.Platform == PlatformID.Unix; - // return System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Linux); } } @@ -252,21 +251,7 @@ public class Runtime #else #error You must define one of PYTHON34 to PYTHON37 or PYTHON27 #endif - // TODO : ideally find a way to do this without having to rename the dylib on Mac/Linux - /* private static string _dllPostFix - { - get - { - if (OSType.IsWindows) - { - return _pyver; - } - // Linux/macOS use dotted version string - return _pyversion; - } - } - internal const string dllBase = "python" + _dllPostFix;*/ internal const string dllBase = "python" + _pyver; From 678afef1a73ad0f8a6dcb634c2cea726b18609f2 Mon Sep 17 00:00:00 2001 From: vkovec Date: Tue, 7 Aug 2018 14:32:36 -0400 Subject: [PATCH 05/10] use UCS2 --- src/runtime/Python.Runtime.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 1fea78082..1ef20f276 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -32,6 +32,7 @@ PYTHON2;PYTHON27;UCS4 true pdbonly + PYTHON2;PYTHON27;UCS2 PYTHON3;PYTHON36;UCS4 From fe9bce4b737723a23ed93bad66adedd0838517e6 Mon Sep 17 00:00:00 2001 From: vkovec Date: Tue, 7 Aug 2018 14:33:22 -0400 Subject: [PATCH 06/10] remove commented out #if --- src/runtime/runtime.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index cd31607b3..30c718b46 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -418,8 +418,7 @@ internal static void Initialize() dllLocal = NativeMethods.LoadLibrary(_PythonDll); } _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); - - //#if !(MONO_LINUX || MONO_OSX) + if (!(OSType.IsLinux || OSType.IsOSX)) { if (dllLocal != IntPtr.Zero) From edd5ba4923613b7d0b96ff4128eb07b5cd4987eb Mon Sep 17 00:00:00 2001 From: vkovec Date: Tue, 7 Aug 2018 14:52:48 -0400 Subject: [PATCH 07/10] use .NET framework 4.7.1 instead of 4 - easier to check which OS we are on --- src/runtime/Python.Runtime.csproj | 12 ++++++++++-- src/runtime/runtime.cs | 8 +++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 1ef20f276..e2f7b0839 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,5 +1,5 @@  - + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Runtime bin\Python.Runtime.xml bin\ - v4.0 + v4.7.1 1591 ..\..\ @@ -33,45 +33,53 @@ true pdbonly PYTHON2;PYTHON27;UCS2 + false PYTHON3;PYTHON36;UCS4 true pdbonly + false true PYTHON2;PYTHON27;UCS4;TRACE;DEBUG false full + false true PYTHON3;PYTHON36;UCS4;TRACE;DEBUG false full + false PYTHON2;PYTHON27;UCS2 true pdbonly + false PYTHON3;PYTHON36;UCS2 true pdbonly + false true PYTHON2;PYTHON27;UCS2;TRACE;DEBUG false full + false true PYTHON3;PYTHON36;UCS2;TRACE;DEBUG false full + false diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 30c718b46..286af456a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -7,12 +7,11 @@ namespace Python.Runtime { internal static class OSType { - // TODO: find how to differentiate between linux and osx public static bool IsLinux { get { - return Environment.OSVersion.Platform == PlatformID.Unix; + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); } } @@ -20,8 +19,7 @@ public static bool IsWindows { get { - var os = Environment.OSVersion.Platform; - return os != PlatformID.MacOSX && os != PlatformID.Unix; + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows); } } @@ -29,7 +27,7 @@ public static bool IsOSX { get { - return Environment.OSVersion.Platform == PlatformID.MacOSX || Environment.OSVersion.Platform == PlatformID.Unix; + return RuntimeInformation.IsOSPlatform(OSPlatform.OSX); } } } From 300e131ed545d6b5019843272b9f397fe57bff35 Mon Sep 17 00:00:00 2001 From: vkovec Date: Fri, 10 Aug 2018 12:40:36 -0400 Subject: [PATCH 08/10] code review fixes - throw exception if trying to call Mac/Linux specific functions on Windows - fix dllBase path to point into packages --- src/runtime/runtime.cs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 286af456a..28cb30a47 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -39,6 +39,11 @@ private static IntPtr RTLD_DEFAULT { get { + if (OSType.IsWindows) + { + // calling this does not make sense on Windows + throw new Exception(); + } if (OSType.IsOSX) { return new IntPtr(-2); @@ -61,6 +66,11 @@ private static int RTLD_GLOBAL { get { + if (OSType.IsWindows) + { + // calling this does not make sense on Windows + throw new Exception(); + } if (OSType.IsOSX) { return 0x8; @@ -129,24 +139,44 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) public static IntPtr dlopen(String fileName, int flags) { + if (OSType.IsWindows) + { + // shouldn't be calling this function on Windows + throw new Exception(); + } if (OSType.IsOSX) { return dlopen_mac(fileName, flags); } return dlopen_linux(fileName, flags); } private static IntPtr dlsym(IntPtr handle, String symbol) { + if (OSType.IsWindows) + { + // shouldn't be calling this function on Windows + throw new Exception(); + } if (OSType.IsOSX) { return dlsym_mac(handle, symbol); } return dlsym_linux(handle, symbol); } private static int dlclose(IntPtr handle) { + if (OSType.IsWindows) + { + // shouldn't be calling this function on Windows + throw new Exception(); + } if (OSType.IsOSX) { return dlclose_mac(handle); } return dlclose_linux(handle); } private static IntPtr dlerror() { + if (OSType.IsWindows) + { + // shouldn't be calling this function on Windows + throw new Exception(); + } if (OSType.IsOSX) { return dlerror_mac(); } return dlerror_linux(); } @@ -188,7 +218,6 @@ private static IntPtr dlerror() [DllImport(winNativeDll, EntryPoint = "FreeLibrary")] public static extern bool FreeLibrary_win(IntPtr hModule); -//#endif } /// @@ -250,7 +279,7 @@ public class Runtime #error You must define one of PYTHON34 to PYTHON37 or PYTHON27 #endif // TODO : ideally find a way to do this without having to rename the dylib on Mac/Linux - internal const string dllBase = "python" + _pyver; + internal const string dllBase = "Packages/com.unity.scripting.python/Editor/bin/python" + _pyver; #if PYTHON_WITH_PYDEBUG From 35774fe44e0d4ad88f9c591b0ce1920ab6698757 Mon Sep 17 00:00:00 2001 From: lassond <39058799+lassond@users.noreply.github.com> Date: Fri, 10 Aug 2018 13:45:25 -0400 Subject: [PATCH 09/10] Uni 52361 python net crashes when hot reloading for the second time * Reinstalling the original import function on Shutdown * Cleaner registration of the DomainUnload callback * Now unloading the Python library on DomainUnload, providing Python .NET with a fresh interpreter each time the assemblies are reloaded --- src/runtime/clrobject.cs | 2 +- src/runtime/importhook.cs | 7 +++++++ src/runtime/pythonengine.cs | 7 ++----- src/runtime/runtime.cs | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index fb3d0e0d7..502677655 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index bc9ac5eee..4ba68ac9d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -76,6 +76,13 @@ internal static void Shutdown() { Runtime.XDecref(py_clr_module); Runtime.XDecref(root.pyHandle); + + // Re-install the original import function + IntPtr dict = Runtime.PyImport_GetModuleDict(); + IntPtr mod = Runtime.IsPython3 + ? Runtime.PyImport_ImportModule("builtins") + : Runtime.PyDict_GetItemString(dict, "__builtin__"); + Runtime.PyObject_SetAttrString(mod, "__import__", py_import); Runtime.XDecref(py_import); } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index f05d641ed..badeba826 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -172,7 +172,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) if (!initialized) { // Make sure we shut down properly on app domain reload - System.AppDomain.CurrentDomain.DomainUnload += new EventHandler(OnDomainUnload); + System.AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; // Creating the delegateManager MUST happen before Runtime.Initialize // is called. If it happens afterwards, DelegateManager's CodeGenerator @@ -310,9 +310,6 @@ public static void Shutdown() { if (initialized) { - // Make sure we shut down properly on app domain reload - System.AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - PyScopeManager.Global.Clear(); Marshal.FreeHGlobal(_pythonHome); _pythonHome = IntPtr.Zero; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 28cb30a47..b29eb72ca 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -476,6 +476,23 @@ internal static void Shutdown() Exceptions.Shutdown(); ImportHook.Shutdown(); Py_Finalize(); + + // Now unload the Python library from memory and load it again, providing a + // fresh interpreter. This prevents a crash (exception) on second domain reload + // https://stackoverflow.com/questions/2445536/unload-a-dll-loaded-using-dllimport + if (_PythonDll != "__Internal") + { + IntPtr dllLocal = NativeMethods.LoadLibrary(_PythonDll); + + // Twice: a first one for the call to LoadLibrary above, + // a second one for the original call to LoadLibrary (should result in unloading the Python library from memory) + NativeMethods.FreeLibrary(dllLocal); + NativeMethods.FreeLibrary(dllLocal); + + // Here the Python library is supposed to be unloaded. + // Load it again in order to get a fresh interpreter + NativeMethods.LoadLibrary(_PythonDll); + } } // called *without* the GIL acquired by clr._AtExit From 7d02ccb5ab860cbef48d246bf679e3177d0f4869 Mon Sep 17 00:00:00 2001 From: lassond <39058799+lassond@users.noreply.github.com> Date: Fri, 10 Aug 2018 14:42:25 -0400 Subject: [PATCH 10/10] Uni 55658 create standalone demo of second hot reload crash This adds a standalone application that reproduce the thrown exception on second domain unload (System.AppDomainUnloadedException: Attempted to access an unloaded AppDomain) --- hotReloadCrashRepro/App.config | 6 + hotReloadCrashRepro/Program.cs | 139 ++++++++++++++++++ .../Properties/AssemblyInfo.cs | 36 +++++ hotReloadCrashRepro/README.txt | 42 ++++++ .../hotReloadCrashRepro.csproj | 72 +++++++++ hotReloadCrashRepro/theAssembly.cs | 44 ++++++ 6 files changed, 339 insertions(+) create mode 100644 hotReloadCrashRepro/App.config create mode 100644 hotReloadCrashRepro/Program.cs create mode 100644 hotReloadCrashRepro/Properties/AssemblyInfo.cs create mode 100644 hotReloadCrashRepro/README.txt create mode 100644 hotReloadCrashRepro/hotReloadCrashRepro.csproj create mode 100644 hotReloadCrashRepro/theAssembly.cs diff --git a/hotReloadCrashRepro/App.config b/hotReloadCrashRepro/App.config new file mode 100644 index 000000000..731f6de6c --- /dev/null +++ b/hotReloadCrashRepro/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/hotReloadCrashRepro/Program.cs b/hotReloadCrashRepro/Program.cs new file mode 100644 index 000000000..1b9345d81 --- /dev/null +++ b/hotReloadCrashRepro/Program.cs @@ -0,0 +1,139 @@ +using System; +using System.CodeDom.Compiler; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.IO; +using System.Threading.Tasks; + +namespace hotReloadCrashRepro +{ + class Program + { + /// + /// Args goes as follows: + /// 0: The full path to theAssembly.cs + /// + /// + static void Main(string[] args) + { + string pathToTheAssembly = ""; + try + { + // Defaults if args are not specified (standard location when + // building with Visual Studio 2017, using the x64 configuration) + pathToTheAssembly = System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, + @"..\..\..\theAssembly.cs"); + } + catch (Exception) + { + } + + if (args.Length > 0) + { + pathToTheAssembly = args[0]; + } + + // The exception is thrown on the second call to Py_Finalize + // + // First time through, on the Python side we litter some objects + // that Python figures someone still has a reference to, so it + // keeps them around -- leak! + // + // Second time through, Python gc looks at the leaked objects and calls + // tp_traverse on them. But the tp_traverse handler is C# code that got + // destroyed in the domain unload -- crash!) + Assembly theCompiledAssembly = null; + for(int i = 0; i < 2; ++i) { + // Create the domain + System.Console.WriteLine(string.Format("[Program.Main] ===Pass #{0}===",i)); + System.Console.WriteLine(string.Format("[Program.Main] Creating the domain \"My Domain {0}\"",i)); + var domain = AppDomain.CreateDomain(string.Format("My Domain {0}",i)); + + // Build the assembly only once (we reuse the same assembly) + if (i == 0) + { + System.Console.WriteLine("[Program.Main] Building the assembly"); + + // The assembly is compiled as a dll in the same directory as the Program executable + theCompiledAssembly = BuildAssembly(pathToTheAssembly, "TheCompiledAssembly.dll"); + } + + // Create a Proxy object in the new domain, where we want the + // assembly (and Python .NET) to reside + Type type = typeof(Proxy); + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + + // From now on use the Proxy to call into the new assembly + theProxy.InitAssembly(theCompiledAssembly.Location); + theProxy.RunPython(); + + System.Console.WriteLine("[Program.Main] Before Domain Unload"); + AppDomain.Unload(domain); + System.Console.WriteLine("[Program.Main] After Domain Unload"); + + // Validate that the assembly does not exist anymore + try + { + System.Console.WriteLine(string.Format("[Program.Main] The Proxy object is valid ({0}). Unexpected domain unload behavior",theProxy)); + } + catch (Exception) + { + System.Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); + } + } + } + + public class Proxy : MarshalByRefObject + { + static Assembly theAssembly = null; + + public void InitAssembly(string assemblyPath) + { + System.Console.WriteLine(string.Format("[Proxy ] In InitAssembly")); + + theAssembly = Assembly.LoadFile(assemblyPath); + var pythonrunner = theAssembly.GetType("PythonRunner"); + var initMethod = pythonrunner.GetMethod("Init"); + initMethod.Invoke(null, new object[] {}); + } + public void RunPython() + { + System.Console.WriteLine(string.Format("[Proxy ] In RunPython")); + + // Call into the new assembly. Will execute Python code + var pythonrunner = theAssembly.GetType("PythonRunner"); + var runPythonMethod = pythonrunner.GetMethod("RunPython"); + runPythonMethod.Invoke(null, new object[] { }); + } + } + + static System.Reflection.Assembly BuildAssembly(string pathToTheAssembly, string outputAssemblyName) + { + var provider = CodeDomProvider.CreateProvider("CSharp"); + var compilerparams = new CompilerParameters(new string [] {"Python.Runtime.dll"}); + + compilerparams.GenerateExecutable = false; + compilerparams.GenerateInMemory = false; + compilerparams.IncludeDebugInformation = true; + compilerparams.OutputAssembly = outputAssemblyName; + + var results = + provider.CompileAssemblyFromFile(compilerparams, pathToTheAssembly); + if (results.Errors.HasErrors) { + StringBuilder errors = new StringBuilder("Compiler Errors :\r\n"); + foreach (CompilerError error in results.Errors ) + { + errors.AppendFormat("Line {0},{1}\t: {2}\n", + error.Line, error.Column, error.ErrorText); + } + throw new Exception(errors.ToString()); + } else { + return results.CompiledAssembly; + } + } + } +} diff --git a/hotReloadCrashRepro/Properties/AssemblyInfo.cs b/hotReloadCrashRepro/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..d1483c93e --- /dev/null +++ b/hotReloadCrashRepro/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("hotReloadCrashRepro")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("HP Inc.")] +[assembly: AssemblyProduct("hotReloadCrashRepro")] +[assembly: AssemblyCopyright("Copyright © HP Inc. 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("22642faa-c1aa-403e-97f7-6503790b6fe5")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/hotReloadCrashRepro/README.txt b/hotReloadCrashRepro/README.txt new file mode 100644 index 000000000..3965c566a --- /dev/null +++ b/hotReloadCrashRepro/README.txt @@ -0,0 +1,42 @@ +This project will reproduce the Python exception on second domain unload. +Make sure you use a version of Python .NET that calls PythonEngine.Shutdown() on DomainReload (starting with commit ###################) + +How to repro: + +1. Open hotReloadCrashRepro.csproj in Visual Studio 2017 +2. Compile using the same platform as Python.Runtime.dll (e.g. x64) +3. Copy Python.Runtime.dll in the directory where hotReloadCrashRepro.exe is located (e.g. bin\x64\Debug) +4. Run "hotReloadCrashRepro.exe full_path_to_theAssembly.cs + e.g. hotReloadCrashRepro.exe "D:\projects\pythonnet\hotReloadCrashRepro\theAssembly.cs" + +The expected output is: + +[Program.Main] ===Pass #0=== +[Program.Main] Creating the domain "My Domain 0" +[Program.Main] Building the assembly +[Proxy ] In InitAssembly +[theAssembly ] PythonRunner.Init current domain = My Domain 0 +[Proxy ] In RunPython +[theAssembly ] In PythonRunner.RunPython +[Python ] Done +[Program.Main] Before Domain Unload +[theAssembly ] In OnDomainUnload current domain = My Domain 0 +[Program.Main] After Domain Unload +[Program.Main] The Proxy object is not valid anymore, domain unload complete. +[Program.Main] ===Pass #1=== +[Program.Main] Creating the domain "My Domain 1" +[Proxy ] In InitAssembly +[theAssembly ] PythonRunner.Init current domain = My Domain 1 +[Proxy ] In RunPython +[theAssembly ] In PythonRunner.RunPython +[Python ] Done +[Program.Main] Before Domain Unload +[theAssembly ] In OnDomainUnload current domain = My Domain 1 + +Unhandled Exception: System.AppDomainUnloadedException: Attempted to access an unloaded AppDomain. + at Python.Runtime.Runtime.Py_Finalize() + at Python.Runtime.Runtime.Shutdown() + at Python.Runtime.PythonEngine.Shutdown() + at Python.Runtime.PythonEngine.OnDomainUnload(Object sender, EventArgs e) +[Program.Main] After Domain Unload +[Program.Main] The Proxy object is not valid anymore, domain unload complete. \ No newline at end of file diff --git a/hotReloadCrashRepro/hotReloadCrashRepro.csproj b/hotReloadCrashRepro/hotReloadCrashRepro.csproj new file mode 100644 index 000000000..e5a1bd5b7 --- /dev/null +++ b/hotReloadCrashRepro/hotReloadCrashRepro.csproj @@ -0,0 +1,72 @@ + + + + + Debug + AnyCPU + {22642FAA-C1AA-403E-97F7-6503790B6FE5} + Exe + hotReloadCrashRepro + hotReloadCrashRepro + v4.6.1 + 512 + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + bin\x64\Debug\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/hotReloadCrashRepro/theAssembly.cs b/hotReloadCrashRepro/theAssembly.cs new file mode 100644 index 000000000..fa090ca6d --- /dev/null +++ b/hotReloadCrashRepro/theAssembly.cs @@ -0,0 +1,44 @@ +using System; +using Python.Runtime; +using System.Reflection; + +public class DummyClass +{ + static public DummyClass instance = new DummyClass(); + public static DummyClass DummyMethod() + { + return instance; + } +} + +class PythonRunner +{ + static public void Init() + { + System.Console.WriteLine(string.Format("[theAssembly ] PythonRunner.Init current domain = {0}",AppDomain.CurrentDomain.FriendlyName)); + + // Register to domain unload + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + } + + private static void OnDomainUnload(object sender, EventArgs e) + { + System.Console.WriteLine(string.Format("[theAssembly ] In OnDomainUnload current domain = {0}",AppDomain.CurrentDomain.FriendlyName)); + } + + public static void RunPython() { + System.Console.WriteLine("[theAssembly ] In PythonRunner.RunPython"); + using (Py.GIL()) { + try { + var pyScript = + "import clr\n" + + "clr.AddReference('System') \n" + + "print('[Python ] Done')\n"; + + PythonEngine.Exec(pyScript); + } catch(Exception e) { + System.Console.WriteLine(string.Format("Caught exception: {0}",e)); + } + } + } +}