From 3ec7739c15d791385b1c5138ed13d602d2ce4c65 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 3 Nov 2026 07:09:19 -0800 Subject: [PATCH 1/2] Let a configured "on" compile through cmake where SuiteSparse uses cmake SuiteSparse built with MSVC always compiles JIT kernels through cmake, and passes the configured compiler to it as CMAKE_C_COMPILER, which cmake's Visual Studio generator ignores. conda-forge's Windows builds name the build machine's cl.exe, which users rarely have, so the check at init that the configured compiler exists turned a configured "on" into "run" there. Where jit_use_cmake is True, init now looks for cmake on the PATH instead, and probes as before: a failing probe still means "run". UDT work no longer raises the default "run" where jit_use_cmake is True, even where the configured compiler exists (the build machine's cl.exe on a host with that Visual Studio, or conda's clang.exe, which the repair finds): a cold probe through cmake took a median of 10.4 s (2.6 to 56 s) over 81 wave-5 Windows CI jobs, so there only an "on" the user set compiles. Registering C source under that "run" says so, and how to opt in. Tests that need UDT work to raise "run" skip where SuiteSparse compiles only through cmake. The probe's type is named after jit_use_cmake and jit_c_cmake_libs too, so switching to cmake compiles a probe of its own rather than loading the one compiled directly; a test checks that, and that init names the probe as later calls do. scripts/jit_diagnostics.py reports where cmake is and jit_use_cmake, times the diagnostic type's registration, and sets "on" as init does, timing the probe, with burble on: on Windows CI that times a compile through cmake, about the least each new kernel costs there. The user guide and the jit_use_cmake and jit_c_control entries of gb.ss.config say what "on" needs on Windows (cmake and Visual Studio), that UDT work does not raise "run" there, and that a new kernel takes about 10 seconds there. --- docs/user_guide/init.rst | 13 +- docs/user_guide/udt.rst | 4 + graphblas/core/ss/jit_config.py | 79 ++++++++--- graphblas/ss/_core.py | 14 +- graphblas/tests/test_ssjit.py | 227 ++++++++++++++++++++++++++++++-- scripts/jit_diagnostics.py | 43 +++++- 6 files changed, 335 insertions(+), 45 deletions(-) diff --git a/docs/user_guide/init.rst b/docs/user_guide/init.rst index 306b596a4..f8a51be62 100644 --- a/docs/user_guide/init.rst +++ b/docs/user_guide/init.rst @@ -102,8 +102,10 @@ written later to ``gb.ss.config`` is taken as it is, and with a C compiler that work, the first operation that needs a new kernel raises ``JitError``, and SuiteSparse lowers the control to ``"load"``. ``gb.ss.fix_jit_config()`` repairs the compiler setting where it can and checks it. On Windows, SuiteSparse built with MSVC (as conda-forge -builds it) compiles JIT kernels only through cmake, and python-graphblas does not turn the -JIT on there yet, so a configured ``"on"`` becomes ``"run"``. +builds it) compiles JIT kernels only through cmake, which finds the Visual Studio C +compiler itself. There ``"on"`` needs cmake (``conda install cmake``) and Visual Studio or +its Build Tools rather than the configured compiler, and UDT work does not raise the +default ``"run"``. With the JIT on: @@ -123,9 +125,10 @@ With the JIT on: instead, by setting the ``GRAPHBLAS_CACHE_PATH`` environment variable before graphblas initializes. The default will change once SuiteSparse publishes compiled kernels atomically. -- The first use of each new kernel compiles it, typically in 0.2 to 1 second. Kernels are - cached on disk, in ``gb.ss.config["jit_cache_path"]`` (by default under - ``~/.SuiteSparse`` on Linux and macOS), so later runs load them instead. +- The first use of each new kernel compiles it, typically in 0.2 to 1 second, or about 10 + seconds through cmake on Windows. Kernels are cached on disk, in + ``gb.ss.config["jit_cache_path"]`` (by default under ``~/.SuiteSparse``), so later runs + load them instead. - The cache is keyed only on the SuiteSparse:GraphBLAS version, so two builds of one version (a conda build and a source build, say) can load each other's kernels, and get wrong results or errors if their internals differ. Give each build its own cache path. diff --git a/docs/user_guide/udt.rst b/docs/user_guide/udt.rst index 9bb5e803a..cd2f93932 100644 --- a/docs/user_guide/udt.rst +++ b/docs/user_guide/udt.rst @@ -536,6 +536,10 @@ included: - ``gb.binary.ss.register_new`` and its ``unary``, ``indexunary``, ``select``, and ``indexbinary`` siblings, which register an operator from a C definition. +None of these raises it where SuiteSparse compiles through cmake, as it always does +when built with MSVC on Windows: each kernel takes about 10 seconds there, so only an +``'on'`` you set compiles (see :ref:`suitesparse_jit`). + A control you set is kept: ``'off'``, ``'pause'``, ``'load'`` (load and run cached kernels, never compile), and ``'run'`` too when it was set before init (``GRAPHBLAS_JIT_C_CONTROL``, ``gb.config.set``, or a YAML config file). With diff --git a/graphblas/core/ss/jit_config.py b/graphblas/core/ss/jit_config.py index 62d78a44a..2e5e00cc2 100644 --- a/graphblas/core/ss/jit_config.py +++ b/graphblas/core/ss/jit_config.py @@ -23,7 +23,8 @@ from ``graphblas.config["jit_c_control"]`` (``'run'`` unless configured otherwise; see :func:`_set_jit_c_control_at_init`). UDT work raises that default ``'run'`` to ``'on'`` when it needs C compiled, but keeps a ``'run'`` -the user set (:func:`_enable_jit_for_udt`). +the user set, and raises nothing where SuiteSparse compiles through cmake +(:func:`_enable_jit_for_udt`). The trade-offs of the JIT, and what to do when it misbehaves, are in the user guide's initialization page (docs/user_guide/init.rst). """ @@ -34,6 +35,7 @@ import pathlib import platform import re +import shutil import subprocess # ``ffi`` and ``lib`` are imported in the functions that use them: importing them @@ -72,6 +74,19 @@ def _compiler_exists(cfg): return bool(cc) and pathlib.Path(cc).exists() +def _compiler_or_cmake_exists(cfg): + """Whether the C compiler is on disk, or cmake on the PATH where SuiteSparse compiles with it. + + SuiteSparse built with MSVC always compiles through cmake, and passes the + configured compiler to it as ``CMAKE_C_COMPILER``, which cmake's Visual Studio + generator ignores. conda-forge's Windows builds name the build machine's + ``cl.exe``, which users rarely have, so the compiler path says nothing there. + """ + if cfg.get("jit_use_cmake"): + return shutil.which("cmake") is not None + return _compiler_exists(cfg) + + def fix_jit_config(*, use_sysconfig=True, probe=True): """Repair the SuiteSparse:GraphBLAS JIT compiler configuration. @@ -221,9 +236,16 @@ def _strip_mismatched_arch(flags): "jit_c_compiler_flags", "jit_c_linker_flags", "jit_c_libraries", + "jit_c_cmake_libs", + "jit_use_cmake", ) +def _probe_name(cfg): + toolchain = "\n".join(str(cfg.get(key, "")) for key in _TOOLCHAIN_KEYS) + return "_jit_probe_" + hashlib.sha256(toolchain.encode()).hexdigest()[:16] + + def _probe_jit(cfg): """Compile a trivial type with the JIT, or load it from the cache, to check the config works. @@ -238,14 +260,13 @@ def _probe_jit(cfg): It goes through the C API, so nothing is registered on the Python side, and it can run while the backend initializes. Once a kernel is in the cache, a later probe loads it (about 2 ms) instead of compiling (about - 0.3 s). The type is named after the toolchain settings, so a compiler or - flags this cache has not seen compile a probe of their own, rather than - load one that another toolchain left in a shared cache. + 0.3 s). The type is named after the toolchain settings (cmake or not among + them), so a compiler or flags this cache has not seen compile a probe of + their own, rather than load one that another toolchain left in a shared cache. """ from .. import ffi, lib - toolchain = "\n".join(cfg.get(key, "") for key in _TOOLCHAIN_KEYS) - name = "_jit_probe_" + hashlib.sha256(toolchain.encode()).hexdigest()[:16] + name = _probe_name(cfg) defn = f"typedef struct {{ int _probe ; }} {name} ;" type_ptr = ffi.new("GrB_Type*") # Size 0 asks SuiteSparse to get the size from a compiled kernel. @@ -268,8 +289,13 @@ class _CConfig: "jit_c_compiler_flags": "GxB_JIT_C_COMPILER_FLAGS", "jit_c_linker_flags": "GxB_JIT_C_LINKER_FLAGS", "jit_c_libraries": "GxB_JIT_C_LIBRARIES", + "jit_c_cmake_libs": "GxB_JIT_C_CMAKE_LIBS", "jit_cache_path": "GxB_JIT_CACHE_PATH", } + _ints = { + "jit_c_control": "GxB_JIT_C_CONTROL", + "jit_use_cmake": "GxB_JIT_USE_CMAKE", + } def __init__(self): from .. import lib @@ -285,9 +311,9 @@ def __init__(self): def __getitem__(self, key): from .. import ffi, lib - if key == "jit_c_control": + if key in self._ints: val = ffi.new("int32_t*") - info = lib.GxB_Global_Option_get_INT32(lib.GxB_JIT_C_CONTROL, val) + info = lib.GxB_Global_Option_get_INT32(getattr(lib, self._ints[key]), val) else: val = ffi.new("char**") info = lib.GxB_Global_Option_get_CHAR(getattr(lib, self._strings[key]), val) @@ -295,6 +321,8 @@ def __getitem__(self, key): raise RuntimeError(f"Could not get SuiteSparse:GraphBLAS option {key!r}") if key == "jit_c_control": return next(k for k, v in self._controls.items() if v == val[0]) + if key == "jit_use_cmake": + return bool(val[0]) return ffi.string(val[0]).decode() def __setitem__(self, key, value): @@ -309,7 +337,7 @@ def __setitem__(self, key, value): raise RuntimeError(f"Could not set SuiteSparse:GraphBLAS option {key!r}") def get(self, key, default=None): - return self[key] if key == "jit_c_control" or key in self._strings else default + return self[key] if key in self._ints or key in self._strings else default def _cache_is_writable(cfg): @@ -338,8 +366,9 @@ def _set_jit_c_control_at_init(control): control is set to ``'run'`` (where the conda-forge and PyPI builds of SuiteSparse start it), with no error or warning: a library built without the JIT, which clamps every write to ``'run'``; no compiler on disk, even - after :func:`_repair_jit_compiler`; a cache directory it cannot write to; - or a compiler that fails the probe. + after :func:`_repair_jit_compiler`, or no cmake where SuiteSparse compiles + through cmake (:func:`_compiler_or_cmake_exists`); a cache directory it + cannot write to; or a compiler that fails the probe. The probe is load-bearing: from SuiteSparse 9.4 on a failed compile raises ``JitError`` (once; SuiteSparse then drops the control to ``'load'``), and without the probe that would be the first user operation that needs a new @@ -359,7 +388,7 @@ def _set_jit_c_control_at_init(control): if not _compiler_exists(cfg): _repair_jit_compiler(cfg) _jit_enabled_for_udt = False - if _compiler_exists(cfg) and _cache_is_writable(cfg): + if _compiler_or_cmake_exists(cfg) and _cache_is_writable(cfg): cfg["jit_c_control"] = "on" _jit_enabled_for_udt = cfg["jit_c_control"] == "on" and _probe_jit(cfg) if not _jit_enabled_for_udt: @@ -428,17 +457,22 @@ def _enable_jit_for_udt(): ``'off'``, ``'pause'``, and ``'load'``, and ``'load'`` is also where SuiteSparse leaves the control after a compile fails, so raising it would retry that compile at every later UDT op (raising ``JitError`` each time on - SuiteSparse 9.4 and later). Returns True iff a kernel armed now will be - compiled. Whether this process can compile at all is probed once and - cached. + SuiteSparse 9.4 and later). Nor is the default raised where SuiteSparse + compiles through cmake (``jit_use_cmake``, always True when it is built + with MSVC): there each kernel took about 10 s on Windows CI, so only an + ``'on'`` the user set compiles, even where the configured compiler exists. + Returns True iff a kernel armed now will be compiled. Whether this process + can compile at all is probed once and cached. """ global _jit_enabled_for_udt cfg = _ss_config() control = cfg.get("jit_c_control") - if control in (None, "off", "pause", "load") or (control == "run" and _user_set_run): - # ``None`` is a library with no JIT at all. The others are honored and - # not cached, so they decide nothing once the control is back at the - # default. + if control in (None, "off", "pause", "load") or ( + control == "run" and (_user_set_run or cfg.get("jit_use_cmake")) + ): + # ``None`` is a library with no JIT at all. The others, and the default + # 'run' where SuiteSparse compiles through cmake, are honored and not + # cached, so they decide nothing once the control is back at the default. return False if _jit_enabled_for_udt is None: if not jit_compiler_is_usable(): @@ -519,6 +553,13 @@ def _compiling_c_source(name): "or None for the default 'run', which python-graphblas raises to 'on' " "for C source." ) + elif after == before == "run" and cfg.get("jit_use_cmake"): + reason = ( + "jit_c_control is still 'run', which neither compiles nor loads: " + "SuiteSparse compiles through cmake here, which python-graphblas " + "does not start unasked (or SuiteSparse was built without the JIT). To " + "compile it through cmake, set gb.ss.config['jit_c_control'] = 'on'." + ) elif after == before == "run": reason = ( "jit_c_control is still 'run', which neither compiles nor loads: " diff --git a/graphblas/ss/_core.py b/graphblas/ss/_core.py index 9fd6eecb6..b8394c3ce 100644 --- a/graphblas/ss/_core.py +++ b/graphblas/ss/_core.py @@ -146,14 +146,16 @@ class GlobalConfig(BaseConfig): Only available for SuiteSparse:GraphBLAS >=8. python-graphblas sets it at init from ``graphblas.config["jit_c_control"]``, or to "run" when that is not set, and UDT work raises that default "run" to - "on" when it needs a kernel compiled. A "run" configured there is kept, and - so is one set here over another configured value; with nothing configured, - "run" set here is the default again, as None is in any case. The user - guide's initialization page covers the trade-offs, and when "on" is safe. + "on" when it needs a kernel compiled, unless jit_use_cmake is True. A "run" + configured there is kept, and so is one set here over another configured + value; with nothing configured, "run" set here is the default again, as + None is in any case. The user guide's initialization page covers the + trade-offs, and when "on" is safe. jit_use_cmake : bool Whether to use cmake to compile the JIT kernels. - Always True for SuiteSparse built with MSVC (on Windows), where - python-graphblas does not turn the JIT on yet. + Always True for SuiteSparse built with MSVC (on Windows), where a + configured "on" needs cmake and Visual Studio, not the configured + compiler. Where it is True, UDT work does not raise "run" to "on". Only available for SuiteSparse:GraphBLAS >=8 jit_c_compiler_name : str C compiler for JIT kernels. diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index 9cfe880a5..e1695f376 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -1,5 +1,6 @@ import contextlib import os +import shutil import subprocess import sys import sysconfig @@ -127,6 +128,44 @@ def _jit_mode(mode): _JIT_WORKS_AT_IMPORT = _jit_works_here() +def _compiles_only_through_cmake(): + """Whether SuiteSparse is built with MSVC (Windows), which compiles only through cmake. + + It keeps ``jit_use_cmake`` True whatever is set. + """ + if _IS_SSGB7: + return False + cfg = gb.ss.config + prev_cmake = cfg["jit_use_cmake"] + cfg["jit_use_cmake"] = False + try: + return cfg["jit_use_cmake"] + finally: + cfg["jit_use_cmake"] = prev_cmake + + +_CMAKE_ONLY = _compiles_only_through_cmake() +# UDT work raises the default 'run' only where SuiteSparse compiles without cmake. +_UDT_WORK_RAISES_RUN = _JIT_WORKS_AT_IMPORT and not _CMAKE_ONLY + + +def _jit_through_cmake_here(): + """Whether SuiteSparse has the JIT and compiles only through cmake, and cmake is on the PATH. + + One built without the JIT keeps ``jit_c_control`` at ``'run'``. Here + ``'on'`` set at init probes through cmake, which finds the Visual Studio + compiler itself. + """ + if not _CMAKE_ONLY or shutil.which("cmake") is None: + return False + with _jit_mode("on"): + return gb.ss.config["jit_c_control"] == "on" + + +_JIT_THROUGH_CMAKE = _jit_through_cmake_here() +_CMAKE_PROBE_FAILED = "cmake could not build the probe: is Visual Studio's C compiler installed?" + + def _run_in_fresh_process(script, jit_c_control=None, **extra_env): """Run ``script`` in a new interpreter on this tree; return its ``key: value`` lines. @@ -209,8 +248,9 @@ def test_ss_import_leaves_jit_c_control_alone(jit_c_control): if jit_c_control != "on": expected = getattr(gb.core.lib, f"GxB_JIT_{(jit_c_control or 'run').upper()}") assert lines["before"] == str(expected), report - elif _JIT_WORKS_AT_IMPORT: - assert lines["before"] == str(gb.core.lib.GxB_JIT_ON), report + elif _JIT_WORKS_AT_IMPORT or _JIT_THROUGH_CMAKE: + hint = "" if _JIT_WORKS_AT_IMPORT else _CMAKE_PROBE_FAILED + "\n" + assert lines["before"] == str(gb.core.lib.GxB_JIT_ON), hint + report else: # psg's wheels (built without the JIT), or no working C compiler assert lines["before"] == str(gb.core.lib.GxB_JIT_RUN), report @@ -288,11 +328,13 @@ def test_set_jit_c_control_at_init(monkeypatch): _set_jit_c_control_at_init("on") assert cfg["jit_c_control"] == "run" assert jit_config._jit_enabled_for_udt is False - # No compiler on disk and nothing to repair it from: no probe at all. + # No compiler on disk and nothing to repair it from (nor cmake, where + # SuiteSparse compiles with it): no probe at all. cfg["jit_c_compiler_name"] = "/nonexistent/path/to/cc" probes = [] monkeypatch.setattr(jit_config, "_repair_jit_compiler", lambda cfg: None) monkeypatch.setattr(jit_config, "_probe_jit", probes.append) + monkeypatch.setattr(shutil, "which", lambda cmd: None) _set_jit_c_control_at_init("on") assert cfg["jit_c_control"] == "run" assert jit_config._jit_enabled_for_udt is False @@ -302,6 +344,104 @@ def test_set_jit_c_control_at_init(monkeypatch): cfg["jit_c_control"] = prev_control +@pytest.mark.skipif("_IS_SSGB7") +def test_set_jit_c_control_at_init_looks_for_cmake(monkeypatch): + """Where SuiteSparse compiles through cmake, an explicit ``'on'`` needs cmake, not the compiler. + + SuiteSparse built with MSVC always does, and passes the configured compiler + to cmake, whose Visual Studio generator ignores it; conda-forge's Windows + builds name the build machine's ``cl.exe``. ``jit_use_cmake`` is set here so + that this runs on every build that lets it be set. + """ + from graphblas.core.ss import jit_config + from graphblas.core.ss.jit_config import _set_jit_c_control_at_init + + cfg = gb.ss.config + prev = {key: cfg[key] for key in ["jit_c_control", "jit_c_compiler_name", "jit_use_cmake"]} + for name in ["_jit_enabled_for_udt", "_configured_jit_c_control", "_user_set_run"]: + monkeypatch.setattr(jit_config, name, getattr(jit_config, name)) + probes = [] + + def probe(cfg): + probes.append(cfg) + return True + + monkeypatch.setattr(jit_config, "_repair_jit_compiler", lambda cfg: None) + monkeypatch.setattr(jit_config, "_cache_is_writable", lambda cfg: True) + monkeypatch.setattr(jit_config, "_probe_jit", probe) + try: + cfg["jit_c_control"] = "on" + has_jit = cfg["jit_c_control"] == "on" # a library built without it keeps 'run' + cfg["jit_c_compiler_name"] = "/nonexistent/path/to/cc" + cfg["jit_use_cmake"] = False + if not cfg["jit_use_cmake"]: # not on SuiteSparse built with MSVC + # A missing compiler is no probe, whether or not cmake is there. + monkeypatch.setattr(shutil, "which", {"cmake": "/path/to/cmake"}.get) + _set_jit_c_control_at_init("on") + assert cfg["jit_c_control"] == "run" + assert probes == [] + cfg["jit_use_cmake"] = True + if not cfg["jit_use_cmake"]: + pytest.skip("SuiteSparse built with MinGW never compiles through cmake") + monkeypatch.setattr(shutil, "which", lambda cmd: None) + _set_jit_c_control_at_init("on") + assert cfg["jit_c_control"] == "run" + assert probes == [] + monkeypatch.setattr(shutil, "which", {"cmake": "/path/to/cmake"}.get) + _set_jit_c_control_at_init("on") + assert len(probes) == int(has_jit) + assert cfg["jit_c_control"] == ("on" if has_jit else "run") + assert jit_config._jit_enabled_for_udt is has_jit + finally: + for key, val in prev.items(): + cfg[key] = val + + +@pytest.mark.skipif("_IS_SSGB7 or not shutil.which('cmake')") +def test_set_jit_c_control_at_init_compiles_through_cmake(monkeypatch): + """An explicit ``'on'`` compiles the probe through cmake where SuiteSparse uses cmake. + + The probe's type is named after ``jit_use_cmake`` too, so it compiles with + cmake here instead of loading the probe compiled directly. Where SuiteSparse + with the JIT is built with MSVC (Windows), which compiles only through cmake, + ``'on'`` must stick; elsewhere it only has to agree with ``_jit_enabled_for_udt``. + """ + from graphblas.core.ss import jit_config + from graphblas.core.ss.jit_config import _set_jit_c_control_at_init + + cfg = gb.ss.config + prev = {key: cfg[key] for key in ["jit_c_control", "jit_use_cmake"]} + for name in ["_jit_enabled_for_udt", "_configured_jit_c_control", "_user_set_run"]: + monkeypatch.setattr(jit_config, name, getattr(jit_config, name)) + try: + cfg["jit_use_cmake"] = True + if not cfg["jit_use_cmake"]: + pytest.skip("SuiteSparse built with MinGW never compiles through cmake") + _set_jit_c_control_at_init("on") + assert cfg["jit_c_control"] == ("on" if jit_config._jit_enabled_for_udt else "run") + if _JIT_THROUGH_CMAKE: + assert cfg["jit_c_control"] == "on", _CMAKE_PROBE_FAILED + finally: + for key, val in prev.items(): + cfg[key] = val + + +@pytest.mark.skipif("_IS_SSGB7") +def test_probe_is_named_after_the_cmake_settings(): + """A probe compiled one way does not answer for another, and init names it as later calls do.""" + from graphblas.core.ss.jit_config import _TOOLCHAIN_KEYS, _CConfig, _probe_name + + cfg = {key: gb.ss.config[key] for key in _TOOLCHAIN_KEYS} + names = { + _probe_name({**cfg, "jit_use_cmake": False}), + _probe_name({**cfg, "jit_use_cmake": True}), + _probe_name({**cfg, "jit_c_cmake_libs": cfg["jit_c_cmake_libs"] + ";m"}), + } + assert len(names) == 3 + # Init reads the settings through the C API, before ``gb.ss`` can be imported. + assert _probe_name(_CConfig()) == _probe_name(gb.ss.config) + + # Run in a subprocess: this is about the very first call in a process, and by # the time any test executes, plenty of JIT setup has already happened. _FIX_JIT_TWICE_PROBE = """ @@ -388,8 +528,10 @@ def test_ss_register_new_enables_the_jit(): ``gb.ss`` stopped raising the control at import, nothing else does it first. """ - if not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; nothing can compile") + if not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit("JIT not compiled in or not working here; nothing can compile") + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") lines, report = _run_in_fresh_process(_SS_REGISTER_NEW_PROBE) kinds = ["unary", "binary", "indexunary", "select"] kinds += ["indexbinary"] if _has_idxbinop else [] @@ -415,6 +557,7 @@ def test_ss_register_new_says_why_the_jit_cannot_build_it(monkeypatch): type_name = f"why_t_{suffix}" tdef = f"typedef struct {{ int64_t why_a ; }} {type_name} ;" prev = gb.ss.config["jit_c_control"] + prev_cmake = gb.ss.config["jit_use_cmake"] try: for control, match in [ ("off", "jit_c_control is 'off', so SuiteSparse will not compile it"), @@ -433,9 +576,16 @@ def test_ss_register_new_says_why_the_jit_cannot_build_it(monkeypatch): gb.ss.config["jit_c_control"] = None with pytest.raises(NullPointer, match="jit_c_control is still 'run'"): unary.ss.register_new(op_name, cdef, "FP64", "FP64") + # So does one where SuiteSparse compiles through cmake, which can compile. + gb.ss.config["jit_use_cmake"] = True + if gb.ss.config["jit_use_cmake"]: # not on SuiteSparse built with MinGW + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", True) + with pytest.raises(NullPointer, match="compiles through cmake here"): + unary.ss.register_new(op_name, cdef, "FP64", "FP64") + gb.ss.config["jit_use_cmake"] = prev_cmake # A probe that fails in this very call leaves 'load' before the op is created. gb.ss.config["jit_c_control"] = "on" - if gb.ss.config["jit_c_control"] == "on": + if gb.ss.config["jit_c_control"] == "on" and not gb.ss.config["jit_use_cmake"]: def failed_probe(cfg): cfg["jit_c_control"] = "load" @@ -448,6 +598,7 @@ def failed_probe(cfg): with pytest.raises(NullPointer, match="the C compiler does not work here"): unary.ss.register_new(op_name, cdef, "FP64", "FP64") finally: + gb.ss.config["jit_use_cmake"] = prev_cmake gb.ss.config["jit_c_control"] = prev @@ -552,8 +703,10 @@ def test_udt_work_raises_only_the_default_run(form, env_control, kept, tmp_path) control. ``'off'``, ``'pause'`` and ``'load'`` were kept before the default was told apart. """ - if not kept and not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; nothing raises to 'on'") + if not kept and not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit("JIT not compiled in or not working here; nothing raises to 'on'") + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") extra_env = {} if form == "YAML file": (tmp_path / "jit.yaml").write_text("jit_c_control: run\n") @@ -649,6 +802,50 @@ def test_enable_jit_for_udt_re_raises_a_lowered_control(monkeypatch): assert fake["jit_c_control"] == "run" +@pytest.mark.skipif("_IS_SSGB7") +def test_enable_jit_for_udt_leaves_run_alone_where_suitesparse_uses_cmake(monkeypatch): + """UDT work does not raise the default ``'run'`` where SuiteSparse compiles through cmake. + + A kernel through cmake took about 10 s on Windows CI, so only an ``'on'`` the + user set compiles there. That holds even where the configured compiler + exists: the build machine's ``cl.exe`` on a host with that Visual Studio, or + conda's ``clang.exe``, which the repair finds. Driven through a stand-in + config, so it runs on every host, whether SuiteSparse was built with MSVC or not. + """ + from graphblas.core.ss import jit_config + + probes = [] + + def probe(cfg): + probes.append(cfg) + return True + + fake = {"jit_use_cmake": True} + monkeypatch.setattr(jit_config, "_ss_config", lambda: fake) + monkeypatch.setattr(jit_config, "_user_set_run", False) + monkeypatch.setattr(jit_config, "jit_compiler_is_usable", lambda: True) + monkeypatch.setattr(jit_config, "_cache_is_writable", lambda cfg: True) + monkeypatch.setattr(jit_config, "_probe_jit", probe) + # Not probed yet, and probed by a configured 'on' that has gone back to 'run' since. + for enabled in [None, True]: + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", enabled) + fake["jit_c_control"] = "run" + assert _enable_jit_for_udt() is False + assert fake["jit_c_control"] == "run" + assert jit_config._jit_enabled_for_udt is enabled + assert probes == [] + # An 'on' the user set compiles, as anywhere. + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", None) + fake["jit_c_control"] = "on" + assert _enable_jit_for_udt() is True + assert len(probes) == 1 + # Where SuiteSparse compiles without cmake, the same default is raised. + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", None) + fake.update(jit_c_control="run", jit_use_cmake=False) + assert _enable_jit_for_udt() is True + assert fake["jit_c_control"] == "on" + + @pytest.mark.skipif("_IS_SSGB7") def test_enable_jit_for_udt_does_not_cache_an_opt_out(monkeypatch): """An ``'off'`` at the first UDT op must not decide the rest of the process. @@ -696,8 +893,12 @@ def test_enable_jit_for_udt_re_raises_the_live_control(monkeypatch): nothing is configured; written over a configured ``'on'``, it is the user's, and kept. None is the default either way. """ - if not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; the control need not hold 'on'") + if not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit( + "JIT not compiled in or not working here; the control need not hold 'on'" + ) + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") from graphblas.core.ss import jit_config @@ -741,8 +942,10 @@ def test_failed_compile_is_not_retried_at_the_next_arming(): next kernel, and on SuiteSparse 9.4 and later raise ``JitError`` again, with ``'off'`` the only way out. """ - if not _JIT_WORKS_AT_IMPORT: - skip_without_jit("JIT not compiled in or not working here; nothing compiles to fail") + if not _UDT_WORK_RAISES_RUN: + if not _JIT_WORKS_AT_IMPORT: + skip_without_jit("JIT not compiled in or not working here; nothing compiles to fail") + pytest.skip("UDT work does not raise 'run' here: it compiles only through cmake") from graphblas.core.ss import jit_config from graphblas.exceptions import JitError diff --git a/scripts/jit_diagnostics.py b/scripts/jit_diagnostics.py index 6959bfb2d..d090c2182 100755 --- a/scripts/jit_diagnostics.py +++ b/scripts/jit_diagnostics.py @@ -12,6 +12,7 @@ import shutil import sys import sysconfig +import time def main(): @@ -52,6 +53,8 @@ def main(): conda_path = pathlib.Path(conda) / "bin" / name if conda else None conda_exists = conda_path.exists() if conda_path else False print(f" {name}: which={which}, conda={'yes' if conda_exists else 'no'}") + # SuiteSparse built with MSVC (Windows) compiles only through cmake. + print(f" cmake: which={shutil.which('cmake')}") # Import graphblas try: @@ -71,6 +74,7 @@ def main(): print("\n--- GraphBLAS JIT settings after graphblas init ---") for key in [ "jit_c_control", + "jit_use_cmake", "jit_c_compiler_name", "jit_c_compiler_flags", "jit_c_linker_flags", @@ -112,8 +116,8 @@ def main(): # Try the fix print("\n--- Attempting gb.ss.fix_jit_config ---") result = gb.ss.fix_jit_config() - # True=JIT working, False=probe failed, None=no conda env - desc = "working" if result is True else "probe failed" if result is False else "no conda" + descs = {True: "working", False: "probe failed", None: "no compiler found to repair from"} + desc = descs[result] print(f" Result: {result} ({desc})") if result is True: print(f" Compiler: {gb.ss.config['jit_c_compiler_name']}") @@ -132,12 +136,14 @@ def main(): prev_burble = gb.ss.config["burble"] gb.ss.config["burble"] = True + start = time.perf_counter() dtype = dtypes.ss.register_new( "jit_diag_test", "typedef struct { int val ; } jit_diag_test ;", ) + seconds = time.perf_counter() - start gb.ss.config["burble"] = prev_burble - print(f" SUCCESS: registered type '{dtype.name}'") + print(f" SUCCESS: registered type '{dtype.name}' in {seconds:.2f} s") except Exception as e: print(f" FAILED: {type(e).__name__}: {e}") try: @@ -147,6 +153,37 @@ def main(): except Exception: pass + # What GRAPHBLAS_JIT_C_CONTROL=on does at init: it probes the compiler, or + # cmake where SuiteSparse compiles with it, and falls back to 'run'. A probe + # compiled through cmake times about the least a new kernel costs there. + print("\n--- jit_c_control 'on', as init sets it ---", flush=True) + from graphblas.core.ss import jit_config + + probe_seconds = [] + probe_jit = jit_config._probe_jit + + def timed_probe(cfg): + start = time.perf_counter() + try: + return probe_jit(cfg) + finally: + probe_seconds.append(time.perf_counter() - start) + + jit_config._probe_jit = timed_probe + prev_burble = gb.ss.config["burble"] + gb.ss.config["burble"] = True + try: + jit_config._set_jit_c_control_at_init("on") + finally: + gb.ss.config["burble"] = prev_burble + jit_config._probe_jit = probe_jit + print(f" Result: {gb.ss.config['jit_c_control']}") + via = "cmake" if gb.ss.config["jit_use_cmake"] else "the C compiler" + if probe_seconds: + print(f" Probe through {via}: {probe_seconds[0]:.2f} s (compile, or load if cached)") + else: + print(f" No probe: {via} not found, the cache not writable, or SuiteSparse has no JIT") + # Print final JIT state print("\n--- Final JIT state ---") print(f" jit_c_control: {gb.ss.config['jit_c_control']}") From b090ee33bab4f2c123f11c43d7366283246d6020 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 4 Nov 2026 10:08:01 -0800 Subject: [PATCH 2/2] Expect "on" through cmake only where a probe compiles, or should test_ssjit's _jit_through_cmake_here decided whether an "on" set at init compiles through cmake by setting "on" and reading it back, which runs no compile. On Windows with psg's 8.0.2.1 wheel, built with MinGW, SuiteSparse compiles through cmake but passes gcc-style flags, which cmake's Visual Studio generator hands to cl.exe, which refuses them (D8021: invalid numeric argument '/Wundef'). Init rightly keeps "run" there, but the helper said yes, so test_ss_import_leaves_jit_c_control_alone[on] and test_set_jit_c_control_at_init_compiles_through_cmake asserted "on" and failed, with a message asking whether Visual Studio was installed, which it was. The helper now asks a real probe, as init does. A failed probe with gcc-style flags (-W..., -std=, -f...) means cmake cannot work here, so the tests check the "run" fallback instead; a failed probe with flags for cl.exe still counts, so the tests report a broken Visual Studio setup. The message no longer blames a missing Visual Studio. A test drives the helper with a stand-in probe and flags on any host. --- graphblas/tests/test_ssjit.py | 69 +++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index e1695f376..690e05c6f 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -150,20 +150,34 @@ def _compiles_only_through_cmake(): def _jit_through_cmake_here(): - """Whether SuiteSparse has the JIT and compiles only through cmake, and cmake is on the PATH. - - One built without the JIT keeps ``jit_c_control`` at ``'run'``. Here - ``'on'`` set at init probes through cmake, which finds the Visual Studio - compiler itself. + """Whether ``'on'`` set at init should compile through cmake here, as SuiteSparse does. + + It must compile only through cmake, have the JIT (one built without it keeps + ``jit_c_control`` at ``'run'``), and find cmake on the PATH, which then finds + the Visual Studio compiler itself. Then it asks a real probe, as init does. A + SuiteSparse built with MinGW (psg's 8.0 wheels) passes gcc-style flags, which + cmake's Visual Studio generator hands to cl.exe, which refuses them: there the + probe fails, and ``'on'`` rightly becomes ``'run'``. A failed probe with flags + for cl.exe still counts, so the tests that need ``'on'`` report it. """ + from graphblas.core.ss import jit_config + if not _CMAKE_ONLY or shutil.which("cmake") is None: return False with _jit_mode("on"): - return gb.ss.config["jit_c_control"] == "on" + if gb.ss.config["jit_c_control"] != "on": + return False + if jit_config._probe_jit(gb.ss.config): + return True + flags = gb.ss.config["jit_c_compiler_flags"].split() + return not any(flag.startswith(("-W", "-std=", "-f")) for flag in flags) _JIT_THROUGH_CMAKE = _jit_through_cmake_here() -_CMAKE_PROBE_FAILED = "cmake could not build the probe: is Visual Studio's C compiler installed?" +_CMAKE_PROBE_FAILED = ( + "cmake could not build the probe with Visual Studio's compiler; its error is in the " + "captured output" +) def _run_in_fresh_process(script, jit_c_control=None, **extra_env): @@ -426,6 +440,47 @@ def test_set_jit_c_control_at_init_compiles_through_cmake(monkeypatch): cfg[key] = val +@pytest.mark.skipif("_IS_SSGB7") +def test_jit_through_cmake_here_asks_a_probe(monkeypatch): + """The cmake tests expect ``'on'`` only where a probe compiles, or should have. + + With psg's 8.0 wheel on Windows (built with MinGW), cmake handed the wheel's gcc + flags to cl.exe, which refused them (D8021, ``/Wundef``), so init rightly kept + ``'run'``; but the check only set ``'on'``, which needs no compile, and the + tests asserted ``'on'``. Driven with a stand-in probe on any host. + """ + from graphblas.core.ss import jit_config + + cfg = gb.ss.config + with _jit_mode("on"): + if cfg["jit_c_control"] != "on": + pytest.skip("SuiteSparse built without the JIT") + prev = {key: cfg[key] for key in ["jit_c_control", "jit_c_compiler_flags"]} + monkeypatch.setitem(globals(), "_CMAKE_ONLY", True) + monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + try: + for flags, compiles, expected in [ + ("-Wundef -std=c11 -fexcess-precision=fast", False, False), # MinGW: run is right + ("/O2 /DNDEBUG", False, True), # cl.exe's flags: the failure is reported + ("-Wundef -std=c11", True, True), + ("/O2 /DNDEBUG", True, True), + ]: + cfg["jit_c_compiler_flags"] = flags + monkeypatch.setattr(jit_config, "_probe_jit", lambda cfg, ok=compiles: ok) + assert _jit_through_cmake_here() is expected, (flags, compiles) + assert cfg["jit_c_control"] == prev["jit_c_control"] + # Not only through cmake, or no cmake: nothing to expect + monkeypatch.setattr(jit_config, "_probe_jit", lambda cfg: True) + monkeypatch.setattr(shutil, "which", lambda name: None) + assert _jit_through_cmake_here() is False + monkeypatch.setitem(globals(), "_CMAKE_ONLY", False) + monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + assert _jit_through_cmake_here() is False + finally: + for key, val in prev.items(): + cfg[key] = val + + @pytest.mark.skipif("_IS_SSGB7") def test_probe_is_named_after_the_cmake_settings(): """A probe compiled one way does not answer for another, and init names it as later calls do."""