diff --git a/docs/user_guide/init.rst b/docs/user_guide/init.rst index 306b596a4..f8a51be62 100644 --- a/docs/user_guide/init.rst +++ b/docs/user_guide/init.rst @@ -102,8 +102,10 @@ written later to ``gb.ss.config`` is taken as it is, and with a C compiler that work, the first operation that needs a new kernel raises ``JitError``, and SuiteSparse lowers the control to ``"load"``. ``gb.ss.fix_jit_config()`` repairs the compiler setting where it can and checks it. On Windows, SuiteSparse built with MSVC (as conda-forge -builds it) compiles JIT kernels only through cmake, and python-graphblas does not turn the -JIT on there yet, so a configured ``"on"`` becomes ``"run"``. +builds it) compiles JIT kernels only through cmake, which finds the Visual Studio C +compiler itself. There ``"on"`` needs cmake (``conda install cmake``) and Visual Studio or +its Build Tools rather than the configured compiler, and UDT work does not raise the +default ``"run"``. With the JIT on: @@ -123,9 +125,10 @@ With the JIT on: instead, by setting the ``GRAPHBLAS_CACHE_PATH`` environment variable before graphblas initializes. The default will change once SuiteSparse publishes compiled kernels atomically. -- The first use of each new kernel compiles it, typically in 0.2 to 1 second. Kernels are - cached on disk, in ``gb.ss.config["jit_cache_path"]`` (by default under - ``~/.SuiteSparse`` on Linux and macOS), so later runs load them instead. +- The first use of each new kernel compiles it, typically in 0.2 to 1 second, or about 10 + seconds through cmake on Windows. Kernels are cached on disk, in + ``gb.ss.config["jit_cache_path"]`` (by default under ``~/.SuiteSparse``), so later runs + load them instead. - The cache is keyed only on the SuiteSparse:GraphBLAS version, so two builds of one version (a conda build and a source build, say) can load each other's kernels, and get wrong results or errors if their internals differ. Give each build its own cache path. diff --git a/docs/user_guide/udt.rst b/docs/user_guide/udt.rst index 9bb5e803a..cd2f93932 100644 --- a/docs/user_guide/udt.rst +++ b/docs/user_guide/udt.rst @@ -536,6 +536,10 @@ included: - ``gb.binary.ss.register_new`` and its ``unary``, ``indexunary``, ``select``, and ``indexbinary`` siblings, which register an operator from a C definition. +None of these raises it where SuiteSparse compiles through cmake, as it always does +when built with MSVC on Windows: each kernel takes about 10 seconds there, so only an +``'on'`` you set compiles (see :ref:`suitesparse_jit`). + A control you set is kept: ``'off'``, ``'pause'``, ``'load'`` (load and run cached kernels, never compile), and ``'run'`` too when it was set before init (``GRAPHBLAS_JIT_C_CONTROL``, ``gb.config.set``, or a YAML config file). With diff --git a/graphblas/core/ss/jit_config.py b/graphblas/core/ss/jit_config.py index 62d78a44a..2e5e00cc2 100644 --- a/graphblas/core/ss/jit_config.py +++ b/graphblas/core/ss/jit_config.py @@ -23,7 +23,8 @@ from ``graphblas.config["jit_c_control"]`` (``'run'`` unless configured otherwise; see :func:`_set_jit_c_control_at_init`). UDT work raises that default ``'run'`` to ``'on'`` when it needs C compiled, but keeps a ``'run'`` -the user set (:func:`_enable_jit_for_udt`). +the user set, and raises nothing where SuiteSparse compiles through cmake +(:func:`_enable_jit_for_udt`). The trade-offs of the JIT, and what to do when it misbehaves, are in the user guide's initialization page (docs/user_guide/init.rst). """ @@ -34,6 +35,7 @@ import pathlib import platform import re +import shutil import subprocess # ``ffi`` and ``lib`` are imported in the functions that use them: importing them @@ -72,6 +74,19 @@ def _compiler_exists(cfg): return bool(cc) and pathlib.Path(cc).exists() +def _compiler_or_cmake_exists(cfg): + """Whether the C compiler is on disk, or cmake on the PATH where SuiteSparse compiles with it. + + SuiteSparse built with MSVC always compiles through cmake, and passes the + configured compiler to it as ``CMAKE_C_COMPILER``, which cmake's Visual Studio + generator ignores. conda-forge's Windows builds name the build machine's + ``cl.exe``, which users rarely have, so the compiler path says nothing there. + """ + if cfg.get("jit_use_cmake"): + return shutil.which("cmake") is not None + return _compiler_exists(cfg) + + def fix_jit_config(*, use_sysconfig=True, probe=True): """Repair the SuiteSparse:GraphBLAS JIT compiler configuration. @@ -221,9 +236,16 @@ def _strip_mismatched_arch(flags): "jit_c_compiler_flags", "jit_c_linker_flags", "jit_c_libraries", + "jit_c_cmake_libs", + "jit_use_cmake", ) +def _probe_name(cfg): + toolchain = "\n".join(str(cfg.get(key, "")) for key in _TOOLCHAIN_KEYS) + return "_jit_probe_" + hashlib.sha256(toolchain.encode()).hexdigest()[:16] + + def _probe_jit(cfg): """Compile a trivial type with the JIT, or load it from the cache, to check the config works. @@ -238,14 +260,13 @@ def _probe_jit(cfg): It goes through the C API, so nothing is registered on the Python side, and it can run while the backend initializes. Once a kernel is in the cache, a later probe loads it (about 2 ms) instead of compiling (about - 0.3 s). The type is named after the toolchain settings, so a compiler or - flags this cache has not seen compile a probe of their own, rather than - load one that another toolchain left in a shared cache. + 0.3 s). The type is named after the toolchain settings (cmake or not among + them), so a compiler or flags this cache has not seen compile a probe of + their own, rather than load one that another toolchain left in a shared cache. """ from .. import ffi, lib - toolchain = "\n".join(cfg.get(key, "") for key in _TOOLCHAIN_KEYS) - name = "_jit_probe_" + hashlib.sha256(toolchain.encode()).hexdigest()[:16] + name = _probe_name(cfg) defn = f"typedef struct {{ int _probe ; }} {name} ;" type_ptr = ffi.new("GrB_Type*") # Size 0 asks SuiteSparse to get the size from a compiled kernel. @@ -268,8 +289,13 @@ class _CConfig: "jit_c_compiler_flags": "GxB_JIT_C_COMPILER_FLAGS", "jit_c_linker_flags": "GxB_JIT_C_LINKER_FLAGS", "jit_c_libraries": "GxB_JIT_C_LIBRARIES", + "jit_c_cmake_libs": "GxB_JIT_C_CMAKE_LIBS", "jit_cache_path": "GxB_JIT_CACHE_PATH", } + _ints = { + "jit_c_control": "GxB_JIT_C_CONTROL", + "jit_use_cmake": "GxB_JIT_USE_CMAKE", + } def __init__(self): from .. import lib @@ -285,9 +311,9 @@ def __init__(self): def __getitem__(self, key): from .. import ffi, lib - if key == "jit_c_control": + if key in self._ints: val = ffi.new("int32_t*") - info = lib.GxB_Global_Option_get_INT32(lib.GxB_JIT_C_CONTROL, val) + info = lib.GxB_Global_Option_get_INT32(getattr(lib, self._ints[key]), val) else: val = ffi.new("char**") info = lib.GxB_Global_Option_get_CHAR(getattr(lib, self._strings[key]), val) @@ -295,6 +321,8 @@ def __getitem__(self, key): raise RuntimeError(f"Could not get SuiteSparse:GraphBLAS option {key!r}") if key == "jit_c_control": return next(k for k, v in self._controls.items() if v == val[0]) + if key == "jit_use_cmake": + return bool(val[0]) return ffi.string(val[0]).decode() def __setitem__(self, key, value): @@ -309,7 +337,7 @@ def __setitem__(self, key, value): raise RuntimeError(f"Could not set SuiteSparse:GraphBLAS option {key!r}") def get(self, key, default=None): - return self[key] if key == "jit_c_control" or key in self._strings else default + return self[key] if key in self._ints or key in self._strings else default def _cache_is_writable(cfg): @@ -338,8 +366,9 @@ def _set_jit_c_control_at_init(control): control is set to ``'run'`` (where the conda-forge and PyPI builds of SuiteSparse start it), with no error or warning: a library built without the JIT, which clamps every write to ``'run'``; no compiler on disk, even - after :func:`_repair_jit_compiler`; a cache directory it cannot write to; - or a compiler that fails the probe. + after :func:`_repair_jit_compiler`, or no cmake where SuiteSparse compiles + through cmake (:func:`_compiler_or_cmake_exists`); a cache directory it + cannot write to; or a compiler that fails the probe. The probe is load-bearing: from SuiteSparse 9.4 on a failed compile raises ``JitError`` (once; SuiteSparse then drops the control to ``'load'``), and without the probe that would be the first user operation that needs a new @@ -359,7 +388,7 @@ def _set_jit_c_control_at_init(control): if not _compiler_exists(cfg): _repair_jit_compiler(cfg) _jit_enabled_for_udt = False - if _compiler_exists(cfg) and _cache_is_writable(cfg): + if _compiler_or_cmake_exists(cfg) and _cache_is_writable(cfg): cfg["jit_c_control"] = "on" _jit_enabled_for_udt = cfg["jit_c_control"] == "on" and _probe_jit(cfg) if not _jit_enabled_for_udt: @@ -428,17 +457,22 @@ def _enable_jit_for_udt(): ``'off'``, ``'pause'``, and ``'load'``, and ``'load'`` is also where SuiteSparse leaves the control after a compile fails, so raising it would retry that compile at every later UDT op (raising ``JitError`` each time on - SuiteSparse 9.4 and later). Returns True iff a kernel armed now will be - compiled. Whether this process can compile at all is probed once and - cached. + SuiteSparse 9.4 and later). Nor is the default raised where SuiteSparse + compiles through cmake (``jit_use_cmake``, always True when it is built + with MSVC): there each kernel took about 10 s on Windows CI, so only an + ``'on'`` the user set compiles, even where the configured compiler exists. + Returns True iff a kernel armed now will be compiled. Whether this process + can compile at all is probed once and cached. """ global _jit_enabled_for_udt cfg = _ss_config() control = cfg.get("jit_c_control") - if control in (None, "off", "pause", "load") or (control == "run" and _user_set_run): - # ``None`` is a library with no JIT at all. The others are honored and - # not cached, so they decide nothing once the control is back at the - # default. + if control in (None, "off", "pause", "load") or ( + control == "run" and (_user_set_run or cfg.get("jit_use_cmake")) + ): + # ``None`` is a library with no JIT at all. The others, and the default + # 'run' where SuiteSparse compiles through cmake, are honored and not + # cached, so they decide nothing once the control is back at the default. return False if _jit_enabled_for_udt is None: if not jit_compiler_is_usable(): @@ -519,6 +553,13 @@ def _compiling_c_source(name): "or None for the default 'run', which python-graphblas raises to 'on' " "for C source." ) + elif after == before == "run" and cfg.get("jit_use_cmake"): + reason = ( + "jit_c_control is still 'run', which neither compiles nor loads: " + "SuiteSparse compiles through cmake here, which python-graphblas " + "does not start unasked (or SuiteSparse was built without the JIT). To " + "compile it through cmake, set gb.ss.config['jit_c_control'] = 'on'." + ) elif after == before == "run": reason = ( "jit_c_control is still 'run', which neither compiles nor loads: " diff --git a/graphblas/ss/_core.py b/graphblas/ss/_core.py index 9fd6eecb6..b8394c3ce 100644 --- a/graphblas/ss/_core.py +++ b/graphblas/ss/_core.py @@ -146,14 +146,16 @@ class GlobalConfig(BaseConfig): Only available for SuiteSparse:GraphBLAS >=8. python-graphblas sets it at init from ``graphblas.config["jit_c_control"]``, or to "run" when that is not set, and UDT work raises that default "run" to - "on" when it needs a kernel compiled. A "run" configured there is kept, and - so is one set here over another configured value; with nothing configured, - "run" set here is the default again, as None is in any case. The user - guide's initialization page covers the trade-offs, and when "on" is safe. + "on" when it needs a kernel compiled, unless jit_use_cmake is True. A "run" + configured there is kept, and so is one set here over another configured + value; with nothing configured, "run" set here is the default again, as + None is in any case. The user guide's initialization page covers the + trade-offs, and when "on" is safe. jit_use_cmake : bool Whether to use cmake to compile the JIT kernels. - Always True for SuiteSparse built with MSVC (on Windows), where - python-graphblas does not turn the JIT on yet. + Always True for SuiteSparse built with MSVC (on Windows), where a + configured "on" needs cmake and Visual Studio, not the configured + compiler. Where it is True, UDT work does not raise "run" to "on". Only available for SuiteSparse:GraphBLAS >=8 jit_c_compiler_name : str C compiler for JIT kernels. diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index 9cfe880a5..690e05c6f 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -1,5 +1,6 @@ import contextlib import os +import shutil import subprocess import sys import sysconfig @@ -127,6 +128,58 @@ def _jit_mode(mode): _JIT_WORKS_AT_IMPORT = _jit_works_here() +def _compiles_only_through_cmake(): + """Whether SuiteSparse is built with MSVC (Windows), which compiles only through cmake. + + It keeps ``jit_use_cmake`` True whatever is set. + """ + if _IS_SSGB7: + return False + cfg = gb.ss.config + prev_cmake = cfg["jit_use_cmake"] + cfg["jit_use_cmake"] = False + try: + return cfg["jit_use_cmake"] + finally: + cfg["jit_use_cmake"] = prev_cmake + + +_CMAKE_ONLY = _compiles_only_through_cmake() +# UDT work raises the default 'run' only where SuiteSparse compiles without cmake. +_UDT_WORK_RAISES_RUN = _JIT_WORKS_AT_IMPORT and not _CMAKE_ONLY + + +def _jit_through_cmake_here(): + """Whether ``'on'`` set at init should compile through cmake here, as SuiteSparse does. + + It must compile only through cmake, have the JIT (one built without it keeps + ``jit_c_control`` at ``'run'``), and find cmake on the PATH, which then finds + the Visual Studio compiler itself. Then it asks a real probe, as init does. A + SuiteSparse built with MinGW (psg's 8.0 wheels) passes gcc-style flags, which + cmake's Visual Studio generator hands to cl.exe, which refuses them: there the + probe fails, and ``'on'`` rightly becomes ``'run'``. A failed probe with flags + for cl.exe still counts, so the tests that need ``'on'`` report it. + """ + from graphblas.core.ss import jit_config + + if not _CMAKE_ONLY or shutil.which("cmake") is None: + return False + with _jit_mode("on"): + if gb.ss.config["jit_c_control"] != "on": + return False + if jit_config._probe_jit(gb.ss.config): + return True + flags = gb.ss.config["jit_c_compiler_flags"].split() + return not any(flag.startswith(("-W", "-std=", "-f")) for flag in flags) + + +_JIT_THROUGH_CMAKE = _jit_through_cmake_here() +_CMAKE_PROBE_FAILED = ( + "cmake could not build the probe with Visual Studio's compiler; its error is in the " + "captured output" +) + + def _run_in_fresh_process(script, jit_c_control=None, **extra_env): """Run ``script`` in a new interpreter on this tree; return its ``key: value`` lines. @@ -209,8 +262,9 @@ def test_ss_import_leaves_jit_c_control_alone(jit_c_control): if jit_c_control != "on": expected = getattr(gb.core.lib, f"GxB_JIT_{(jit_c_control or 'run').upper()}") assert lines["before"] == str(expected), report - elif _JIT_WORKS_AT_IMPORT: - assert lines["before"] == str(gb.core.lib.GxB_JIT_ON), report + elif _JIT_WORKS_AT_IMPORT or _JIT_THROUGH_CMAKE: + hint = "" if _JIT_WORKS_AT_IMPORT else _CMAKE_PROBE_FAILED + "\n" + assert lines["before"] == str(gb.core.lib.GxB_JIT_ON), hint + report else: # psg's wheels (built without the JIT), or no working C compiler assert lines["before"] == str(gb.core.lib.GxB_JIT_RUN), report @@ -288,11 +342,13 @@ def test_set_jit_c_control_at_init(monkeypatch): _set_jit_c_control_at_init("on") assert cfg["jit_c_control"] == "run" assert jit_config._jit_enabled_for_udt is False - # No compiler on disk and nothing to repair it from: no probe at all. + # No compiler on disk and nothing to repair it from (nor cmake, where + # SuiteSparse compiles with it): no probe at all. cfg["jit_c_compiler_name"] = "/nonexistent/path/to/cc" probes = [] monkeypatch.setattr(jit_config, "_repair_jit_compiler", lambda cfg: None) monkeypatch.setattr(jit_config, "_probe_jit", probes.append) + monkeypatch.setattr(shutil, "which", lambda cmd: None) _set_jit_c_control_at_init("on") assert cfg["jit_c_control"] == "run" assert jit_config._jit_enabled_for_udt is False @@ -302,6 +358,145 @@ def test_set_jit_c_control_at_init(monkeypatch): cfg["jit_c_control"] = prev_control +@pytest.mark.skipif("_IS_SSGB7") +def test_set_jit_c_control_at_init_looks_for_cmake(monkeypatch): + """Where SuiteSparse compiles through cmake, an explicit ``'on'`` needs cmake, not the compiler. + + SuiteSparse built with MSVC always does, and passes the configured compiler + to cmake, whose Visual Studio generator ignores it; conda-forge's Windows + builds name the build machine's ``cl.exe``. ``jit_use_cmake`` is set here so + that this runs on every build that lets it be set. + """ + from graphblas.core.ss import jit_config + from graphblas.core.ss.jit_config import _set_jit_c_control_at_init + + cfg = gb.ss.config + prev = {key: cfg[key] for key in ["jit_c_control", "jit_c_compiler_name", "jit_use_cmake"]} + for name in ["_jit_enabled_for_udt", "_configured_jit_c_control", "_user_set_run"]: + monkeypatch.setattr(jit_config, name, getattr(jit_config, name)) + probes = [] + + def probe(cfg): + probes.append(cfg) + return True + + monkeypatch.setattr(jit_config, "_repair_jit_compiler", lambda cfg: None) + monkeypatch.setattr(jit_config, "_cache_is_writable", lambda cfg: True) + monkeypatch.setattr(jit_config, "_probe_jit", probe) + try: + cfg["jit_c_control"] = "on" + has_jit = cfg["jit_c_control"] == "on" # a library built without it keeps 'run' + cfg["jit_c_compiler_name"] = "/nonexistent/path/to/cc" + cfg["jit_use_cmake"] = False + if not cfg["jit_use_cmake"]: # not on SuiteSparse built with MSVC + # A missing compiler is no probe, whether or not cmake is there. + monkeypatch.setattr(shutil, "which", {"cmake": "/path/to/cmake"}.get) + _set_jit_c_control_at_init("on") + assert cfg["jit_c_control"] == "run" + assert probes == [] + cfg["jit_use_cmake"] = True + if not cfg["jit_use_cmake"]: + pytest.skip("SuiteSparse built with MinGW never compiles through cmake") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + _set_jit_c_control_at_init("on") + assert cfg["jit_c_control"] == "run" + assert probes == [] + monkeypatch.setattr(shutil, "which", {"cmake": "/path/to/cmake"}.get) + _set_jit_c_control_at_init("on") + assert len(probes) == int(has_jit) + assert cfg["jit_c_control"] == ("on" if has_jit else "run") + assert jit_config._jit_enabled_for_udt is has_jit + finally: + for key, val in prev.items(): + cfg[key] = val + + +@pytest.mark.skipif("_IS_SSGB7 or not shutil.which('cmake')") +def test_set_jit_c_control_at_init_compiles_through_cmake(monkeypatch): + """An explicit ``'on'`` compiles the probe through cmake where SuiteSparse uses cmake. + + The probe's type is named after ``jit_use_cmake`` too, so it compiles with + cmake here instead of loading the probe compiled directly. Where SuiteSparse + with the JIT is built with MSVC (Windows), which compiles only through cmake, + ``'on'`` must stick; elsewhere it only has to agree with ``_jit_enabled_for_udt``. + """ + from graphblas.core.ss import jit_config + from graphblas.core.ss.jit_config import _set_jit_c_control_at_init + + cfg = gb.ss.config + prev = {key: cfg[key] for key in ["jit_c_control", "jit_use_cmake"]} + for name in ["_jit_enabled_for_udt", "_configured_jit_c_control", "_user_set_run"]: + monkeypatch.setattr(jit_config, name, getattr(jit_config, name)) + try: + cfg["jit_use_cmake"] = True + if not cfg["jit_use_cmake"]: + pytest.skip("SuiteSparse built with MinGW never compiles through cmake") + _set_jit_c_control_at_init("on") + assert cfg["jit_c_control"] == ("on" if jit_config._jit_enabled_for_udt else "run") + if _JIT_THROUGH_CMAKE: + assert cfg["jit_c_control"] == "on", _CMAKE_PROBE_FAILED + finally: + for key, val in prev.items(): + cfg[key] = val + + +@pytest.mark.skipif("_IS_SSGB7") +def test_jit_through_cmake_here_asks_a_probe(monkeypatch): + """The cmake tests expect ``'on'`` only where a probe compiles, or should have. + + With psg's 8.0 wheel on Windows (built with MinGW), cmake handed the wheel's gcc + flags to cl.exe, which refused them (D8021, ``/Wundef``), so init rightly kept + ``'run'``; but the check only set ``'on'``, which needs no compile, and the + tests asserted ``'on'``. Driven with a stand-in probe on any host. + """ + from graphblas.core.ss import jit_config + + cfg = gb.ss.config + with _jit_mode("on"): + if cfg["jit_c_control"] != "on": + pytest.skip("SuiteSparse built without the JIT") + prev = {key: cfg[key] for key in ["jit_c_control", "jit_c_compiler_flags"]} + monkeypatch.setitem(globals(), "_CMAKE_ONLY", True) + monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + try: + for flags, compiles, expected in [ + ("-Wundef -std=c11 -fexcess-precision=fast", False, False), # MinGW: run is right + ("/O2 /DNDEBUG", False, True), # cl.exe's flags: the failure is reported + ("-Wundef -std=c11", True, True), + ("/O2 /DNDEBUG", True, True), + ]: + cfg["jit_c_compiler_flags"] = flags + monkeypatch.setattr(jit_config, "_probe_jit", lambda cfg, ok=compiles: ok) + assert _jit_through_cmake_here() is expected, (flags, compiles) + assert cfg["jit_c_control"] == prev["jit_c_control"] + # Not only through cmake, or no cmake: nothing to expect + monkeypatch.setattr(jit_config, "_probe_jit", lambda cfg: True) + monkeypatch.setattr(shutil, "which", lambda name: None) + assert _jit_through_cmake_here() is False + monkeypatch.setitem(globals(), "_CMAKE_ONLY", False) + monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + assert _jit_through_cmake_here() is False + finally: + for key, val in prev.items(): + cfg[key] = val + + +@pytest.mark.skipif("_IS_SSGB7") +def test_probe_is_named_after_the_cmake_settings(): + """A probe compiled one way does not answer for another, and init names it as later calls do.""" + from graphblas.core.ss.jit_config import _TOOLCHAIN_KEYS, _CConfig, _probe_name + + cfg = {key: gb.ss.config[key] for key in _TOOLCHAIN_KEYS} + names = { + _probe_name({**cfg, "jit_use_cmake": False}), + _probe_name({**cfg, "jit_use_cmake": True}), + _probe_name({**cfg, "jit_c_cmake_libs": cfg["jit_c_cmake_libs"] + ";m"}), + } + assert len(names) == 3 + # Init reads the settings through the C API, before ``gb.ss`` can be imported. + assert _probe_name(_CConfig()) == _probe_name(gb.ss.config) + + # Run in a subprocess: this is about the very first call in a process, and by # the time any test executes, plenty of JIT setup has already happened. _FIX_JIT_TWICE_PROBE = """ @@ -388,8 +583,10 @@ def test_ss_register_new_enables_the_jit(): ``gb.ss`` stopped raising the control at import, nothing else does it first. """ - if not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; nothing can compile") + if not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit("JIT not compiled in or not working here; nothing can compile") + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") lines, report = _run_in_fresh_process(_SS_REGISTER_NEW_PROBE) kinds = ["unary", "binary", "indexunary", "select"] kinds += ["indexbinary"] if _has_idxbinop else [] @@ -415,6 +612,7 @@ def test_ss_register_new_says_why_the_jit_cannot_build_it(monkeypatch): type_name = f"why_t_{suffix}" tdef = f"typedef struct {{ int64_t why_a ; }} {type_name} ;" prev = gb.ss.config["jit_c_control"] + prev_cmake = gb.ss.config["jit_use_cmake"] try: for control, match in [ ("off", "jit_c_control is 'off', so SuiteSparse will not compile it"), @@ -433,9 +631,16 @@ def test_ss_register_new_says_why_the_jit_cannot_build_it(monkeypatch): gb.ss.config["jit_c_control"] = None with pytest.raises(NullPointer, match="jit_c_control is still 'run'"): unary.ss.register_new(op_name, cdef, "FP64", "FP64") + # So does one where SuiteSparse compiles through cmake, which can compile. + gb.ss.config["jit_use_cmake"] = True + if gb.ss.config["jit_use_cmake"]: # not on SuiteSparse built with MinGW + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", True) + with pytest.raises(NullPointer, match="compiles through cmake here"): + unary.ss.register_new(op_name, cdef, "FP64", "FP64") + gb.ss.config["jit_use_cmake"] = prev_cmake # A probe that fails in this very call leaves 'load' before the op is created. gb.ss.config["jit_c_control"] = "on" - if gb.ss.config["jit_c_control"] == "on": + if gb.ss.config["jit_c_control"] == "on" and not gb.ss.config["jit_use_cmake"]: def failed_probe(cfg): cfg["jit_c_control"] = "load" @@ -448,6 +653,7 @@ def failed_probe(cfg): with pytest.raises(NullPointer, match="the C compiler does not work here"): unary.ss.register_new(op_name, cdef, "FP64", "FP64") finally: + gb.ss.config["jit_use_cmake"] = prev_cmake gb.ss.config["jit_c_control"] = prev @@ -552,8 +758,10 @@ def test_udt_work_raises_only_the_default_run(form, env_control, kept, tmp_path) control. ``'off'``, ``'pause'`` and ``'load'`` were kept before the default was told apart. """ - if not kept and not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; nothing raises to 'on'") + if not kept and not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit("JIT not compiled in or not working here; nothing raises to 'on'") + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") extra_env = {} if form == "YAML file": (tmp_path / "jit.yaml").write_text("jit_c_control: run\n") @@ -649,6 +857,50 @@ def test_enable_jit_for_udt_re_raises_a_lowered_control(monkeypatch): assert fake["jit_c_control"] == "run" +@pytest.mark.skipif("_IS_SSGB7") +def test_enable_jit_for_udt_leaves_run_alone_where_suitesparse_uses_cmake(monkeypatch): + """UDT work does not raise the default ``'run'`` where SuiteSparse compiles through cmake. + + A kernel through cmake took about 10 s on Windows CI, so only an ``'on'`` the + user set compiles there. That holds even where the configured compiler + exists: the build machine's ``cl.exe`` on a host with that Visual Studio, or + conda's ``clang.exe``, which the repair finds. Driven through a stand-in + config, so it runs on every host, whether SuiteSparse was built with MSVC or not. + """ + from graphblas.core.ss import jit_config + + probes = [] + + def probe(cfg): + probes.append(cfg) + return True + + fake = {"jit_use_cmake": True} + monkeypatch.setattr(jit_config, "_ss_config", lambda: fake) + monkeypatch.setattr(jit_config, "_user_set_run", False) + monkeypatch.setattr(jit_config, "jit_compiler_is_usable", lambda: True) + monkeypatch.setattr(jit_config, "_cache_is_writable", lambda cfg: True) + monkeypatch.setattr(jit_config, "_probe_jit", probe) + # Not probed yet, and probed by a configured 'on' that has gone back to 'run' since. + for enabled in [None, True]: + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", enabled) + fake["jit_c_control"] = "run" + assert _enable_jit_for_udt() is False + assert fake["jit_c_control"] == "run" + assert jit_config._jit_enabled_for_udt is enabled + assert probes == [] + # An 'on' the user set compiles, as anywhere. + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", None) + fake["jit_c_control"] = "on" + assert _enable_jit_for_udt() is True + assert len(probes) == 1 + # Where SuiteSparse compiles without cmake, the same default is raised. + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", None) + fake.update(jit_c_control="run", jit_use_cmake=False) + assert _enable_jit_for_udt() is True + assert fake["jit_c_control"] == "on" + + @pytest.mark.skipif("_IS_SSGB7") def test_enable_jit_for_udt_does_not_cache_an_opt_out(monkeypatch): """An ``'off'`` at the first UDT op must not decide the rest of the process. @@ -696,8 +948,12 @@ def test_enable_jit_for_udt_re_raises_the_live_control(monkeypatch): nothing is configured; written over a configured ``'on'``, it is the user's, and kept. None is the default either way. """ - if not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; the control need not hold 'on'") + if not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit( + "JIT not compiled in or not working here; the control need not hold 'on'" + ) + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") from graphblas.core.ss import jit_config @@ -741,8 +997,10 @@ def test_failed_compile_is_not_retried_at_the_next_arming(): next kernel, and on SuiteSparse 9.4 and later raise ``JitError`` again, with ``'off'`` the only way out. """ - if not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; nothing compiles to fail") + if not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit("JIT not compiled in or not working here; nothing compiles to fail") + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") from graphblas.core.ss import jit_config from graphblas.exceptions import JitError diff --git a/scripts/jit_diagnostics.py b/scripts/jit_diagnostics.py index 6959bfb2d..d090c2182 100755 --- a/scripts/jit_diagnostics.py +++ b/scripts/jit_diagnostics.py @@ -12,6 +12,7 @@ import shutil import sys import sysconfig +import time def main(): @@ -52,6 +53,8 @@ def main(): conda_path = pathlib.Path(conda) / "bin" / name if conda else None conda_exists = conda_path.exists() if conda_path else False print(f" {name}: which={which}, conda={'yes' if conda_exists else 'no'}") + # SuiteSparse built with MSVC (Windows) compiles only through cmake. + print(f" cmake: which={shutil.which('cmake')}") # Import graphblas try: @@ -71,6 +74,7 @@ def main(): print("\n--- GraphBLAS JIT settings after graphblas init ---") for key in [ "jit_c_control", + "jit_use_cmake", "jit_c_compiler_name", "jit_c_compiler_flags", "jit_c_linker_flags", @@ -112,8 +116,8 @@ def main(): # Try the fix print("\n--- Attempting gb.ss.fix_jit_config ---") result = gb.ss.fix_jit_config() - # True=JIT working, False=probe failed, None=no conda env - desc = "working" if result is True else "probe failed" if result is False else "no conda" + descs = {True: "working", False: "probe failed", None: "no compiler found to repair from"} + desc = descs[result] print(f" Result: {result} ({desc})") if result is True: print(f" Compiler: {gb.ss.config['jit_c_compiler_name']}") @@ -132,12 +136,14 @@ def main(): prev_burble = gb.ss.config["burble"] gb.ss.config["burble"] = True + start = time.perf_counter() dtype = dtypes.ss.register_new( "jit_diag_test", "typedef struct { int val ; } jit_diag_test ;", ) + seconds = time.perf_counter() - start gb.ss.config["burble"] = prev_burble - print(f" SUCCESS: registered type '{dtype.name}'") + print(f" SUCCESS: registered type '{dtype.name}' in {seconds:.2f} s") except Exception as e: print(f" FAILED: {type(e).__name__}: {e}") try: @@ -147,6 +153,37 @@ def main(): except Exception: pass + # What GRAPHBLAS_JIT_C_CONTROL=on does at init: it probes the compiler, or + # cmake where SuiteSparse compiles with it, and falls back to 'run'. A probe + # compiled through cmake times about the least a new kernel costs there. + print("\n--- jit_c_control 'on', as init sets it ---", flush=True) + from graphblas.core.ss import jit_config + + probe_seconds = [] + probe_jit = jit_config._probe_jit + + def timed_probe(cfg): + start = time.perf_counter() + try: + return probe_jit(cfg) + finally: + probe_seconds.append(time.perf_counter() - start) + + jit_config._probe_jit = timed_probe + prev_burble = gb.ss.config["burble"] + gb.ss.config["burble"] = True + try: + jit_config._set_jit_c_control_at_init("on") + finally: + gb.ss.config["burble"] = prev_burble + jit_config._probe_jit = probe_jit + print(f" Result: {gb.ss.config['jit_c_control']}") + via = "cmake" if gb.ss.config["jit_use_cmake"] else "the C compiler" + if probe_seconds: + print(f" Probe through {via}: {probe_seconds[0]:.2f} s (compile, or load if cached)") + else: + print(f" No probe: {via} not found, the cache not writable, or SuiteSparse has no JIT") + # Print final JIT state print("\n--- Final JIT state ---") print(f" jit_c_control: {gb.ss.config['jit_c_control']}")