diff --git a/README.md b/README.md index 7b157c979..81df4fe64 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ import graphblas as gb # scipy.sparse matrices A = gb.io.from_scipy_sparse(m) -m = gb.io.to_scipy_sparse(m, format="csr") +m = gb.io.to_scipy_sparse(A, format="csr") # networkx graphs A = gb.io.from_networkx(g) diff --git a/docs/_static/matrix.css b/docs/_static/matrix.css index 1937178e5..b773003cd 100644 --- a/docs/_static/matrix.css +++ b/docs/_static/matrix.css @@ -5,7 +5,7 @@ table.matrix { border: 0px; } -/* Disable a horizintal line from the default stylesheet */ +/* Disable a horizontal line from the default stylesheet */ .table.matrix > :not(caption) > * > * { border-bottom-width: 0px; } @@ -48,7 +48,7 @@ table.matrix > tbody tr td:last-of-type { /* prevents empty cells from collapsing, especially empty rows */ table.matrix > tbody tr td:empty::before { - /* basicaly fills empty cells with   */ + /* basically fills empty cells with   */ content: "\00a0\00a0\00a0"; visibility: hidden; } diff --git a/docs/user_guide/operators.rst b/docs/user_guide/operators.rst index 6a66c295c..59a38c791 100644 --- a/docs/user_guide/operators.rst +++ b/docs/user_guide/operators.rst @@ -331,7 +331,7 @@ This facilitates writing more succinct code such as: cur_min(accum=op.min) << op.min_plus(A @ B).reduce_rowwise(op.min) -In the case of name conflicts (ex. binary.min and monoid.min), only one will exist in the the +In the case of name conflicts (ex. binary.min and monoid.min), only one will exist in the ``graphblas.op`` namespace. However, almost all functions which require a specific kind of operator have a mechanism to convert from an identically named operator of a different type. diff --git a/docs/user_guide/udt.rst b/docs/user_guide/udt.rst index d37653250..15cbedbb4 100644 --- a/docs/user_guide/udt.rst +++ b/docs/user_guide/udt.rst @@ -290,13 +290,18 @@ import changes nothing else, so reading ``gb.ss.about`` does not change what later operations compute. ``jit_c_control`` is raised from the SS default ``'run'`` (run cached kernels -only; no compile, no load from disk) to ``'on'`` (compile, load, and run) when -a UDT operator gets C source for SuiteSparse to compile, such as the first -``binary.plus[udt]``, or when ``gb.dtypes.ss.register_new`` registers a type -from a C typedef. Any other setting is kept, including ``'off'``, ``'pause'``, -and ``'load'`` (load and run cached kernels, never compile). SuiteSparse -itself sets ``'load'`` after a compile fails, so keeping it means a failing -compile is tried once, not again at every later UDT operation. +only; no compile, no load from disk) to ``'on'`` (compile, load, and run) by +each of these, since each asks SuiteSparse to compile C source: + +- a UDT operator that gets C source, such as the first ``binary.plus[udt]``; +- ``gb.dtypes.ss.register_new``, which registers a type from a C typedef; +- ``gb.binary.ss.register_new`` and its ``unary``, ``indexunary``, ``select``, + and ``indexbinary`` siblings, which register an operator from a C definition. + +A setting other than ``'run'`` is kept, including ``'off'``, ``'pause'``, and +``'load'`` (load and run cached kernels, never compile). SuiteSparse itself +sets ``'load'`` after a compile fails, so keeping it means a failing compile +is tried once, not again at every later request. Call the helper manually to re-fix or verify:: diff --git a/graphblas/core/dtypes.py b/graphblas/core/dtypes.py index 73b905306..93b3d4d8a 100644 --- a/graphblas/core/dtypes.py +++ b/graphblas/core/dtypes.py @@ -308,7 +308,7 @@ def register_anonymous(dtype, name=None): else: np_repr = np_repr[: lib.GxB_MAX_NAME_LEN] warnings.warn( - f"{msg}. It will use the following name, " + f"{msg} It will use the following name, " f"and the dtype may need to be specified when deserializing: {np_repr}", stacklevel=2, ) diff --git a/graphblas/core/matrix.py b/graphblas/core/matrix.py index 698bf7b4f..4625e4258 100644 --- a/graphblas/core/matrix.py +++ b/graphblas/core/matrix.py @@ -368,7 +368,7 @@ def __sizeof__(self): size = ffi_new("size_t*") check_status(lib.GxB_Matrix_memoryUsage(size, self.gb_obj[0]), self) return size[0] + object.__sizeof__(self) - raise TypeError("Unable to get size of Matrix with backend: {backend}") + raise TypeError(f"Unable to get size of Matrix with backend: {backend}") def isequal(self, other, *, check_dtype=False, **opts): """Check for exact equality (same size, same structure). @@ -377,7 +377,7 @@ def isequal(self, other, *, check_dtype=False, **opts): ---------- other : Matrix The matrix to compare against - check_dtypes : bool + check_dtype : bool If True, also checks that dtypes match Returns @@ -532,7 +532,7 @@ def to_coo(self, dtype=None, *, rows=True, columns=True, values=True, sort=True) Requested dtype for the output values array. rows : bool, default=True Whether to return rows; will return ``None`` for rows if ``False`` - columns :bool, default=True + columns : bool, default=True Whether to return columns; will return ``None`` for columns if ``False`` values : bool, default=True Whether to return values; will return ``None`` for values if ``False`` @@ -1131,15 +1131,15 @@ def from_csc( ---------- indptr : list or np.ndarray Pointers for each column into row_indices and values; ``indptr.size == ncols + 1``. - col_indices : list or np.ndarray - Column indices. + row_indices : list or np.ndarray + Row indices. values : list or np.ndarray or scalar, default 1.0 List of values. If a scalar is provided, all values will be set to this single value. dtype : Data type of the Matrix. If not provided, the values will be inspected to choose an appropriate dtype. nrows : int, optional - Number of rows in the Matrix. If not provided, ``ncols`` is computed + Number of rows in the Matrix. If not provided, ``nrows`` is computed from the maximum row index found in ``row_indices``. ncols : int, optional Number of columns in the Matrix. ncols is computed as ``len(indptr) - 1``. @@ -1529,7 +1529,7 @@ def from_dicts( Parameters ---------- - d : Mapping or Sequence + nested_dicts : Mapping or Sequence The dict-like object to convert. The keys will be cast to uint64 for the indices. dtype : Data type of the Matrix. If not provided, the values will be inspected @@ -2974,7 +2974,7 @@ def setdiag(self, values, k=0, *, mask=None, accum=None, **opts): elif v._size != size: raise DimensionMismatch( f"Dimensions not compatible for assigning length {v._size} Vector " - f"to {k}'th diagonal of Matrix with shape {self._nrows}x{self._ncols}." + f"to {k}'th diagonal of Matrix with shape {self._nrows}x{self._ncols}. " f"The Vector should be size {size}." ) diff --git a/graphblas/core/operator/base.py b/graphblas/core/operator/base.py index 28c2291f8..485857cfb 100644 --- a/graphblas/core/operator/base.py +++ b/graphblas/core/operator/base.py @@ -833,7 +833,7 @@ def __getitem__(self, type_): raise KeyError(f"{self.name} does not work with {type_}") else: return self._typed_ops[type_] - # This is a UDT or is able to operate on UDTs such as `first` any `any` + # This is a UDT or is able to operate on UDTs such as `first` and `any` dtype = lookup_dtype(type_) return self._compile_udt(dtype, dtype) @@ -903,7 +903,7 @@ def _initialize(cls, include_in_ops=True): """Initialize operators for this operator type. include_in_ops determines whether the operators are included in the - ``gb.ops`` namespace in addition to the defined module. + ``gb.op`` namespace in addition to the defined module. """ if cls._initialized: # pragma: no cover (safety) return diff --git a/graphblas/core/operator/binary.py b/graphblas/core/operator/binary.py index 5329b4415..2cdab6eff 100644 --- a/graphblas/core/operator/binary.py +++ b/graphblas/core/operator/binary.py @@ -402,7 +402,7 @@ class BinaryOp(OpBase): """Takes two inputs and returns one output, possibly of a different data type. Built-in and registered BinaryOps are located in the ``graphblas.binary`` namespace - as well as in the ``graphblas.ops`` combined namespace. + as well as in the ``graphblas.op`` combined namespace. """ __slots__ = ( diff --git a/graphblas/core/operator/monoid.py b/graphblas/core/operator/monoid.py index eb3aed6c1..0b02d4226 100644 --- a/graphblas/core/operator/monoid.py +++ b/graphblas/core/operator/monoid.py @@ -226,7 +226,7 @@ class Monoid(OpBase): """Takes two inputs and returns one output, all of the same data type. Built-in and registered Monoids are located in the ``graphblas.monoid`` namespace - as well as in the ``graphblas.ops`` combined namespace. + as well as in the ``graphblas.op`` combined namespace. """ __slots__ = "_binaryop", "_identity", "_is_idempotent" @@ -347,10 +347,10 @@ def register_anonymous(cls, binaryop, identity, name=None, *, is_idempotent=Fals Parameters ---------- - binaryop: BinaryOp or ParameterizedBinaryOp + binaryop : BinaryOp or ParameterizedBinaryOp The binary operator of the monoid, which should be able to use the same dtype for both inputs and the output. - identity: scalar or Mapping + identity : scalar or Mapping The identity of the monoid such that ``op(x, identity) == x`` for any x. ``identity`` may also be a mapping from dtype to scalar. name : str, optional @@ -381,10 +381,10 @@ def register_new(cls, name, binaryop, identity, *, is_idempotent=False, lazy=Fal The name of the operator. This will show up as ``gb.monoid.{name}``. The name may contain periods, ".", which will result in nested objects such as ``gb.monoid.x.y.z`` for name ``"x.y.z"``. - binaryop: BinaryOp or ParameterizedBinaryOp + binaryop : BinaryOp or ParameterizedBinaryOp The binary operator of the monoid, which should be able to use the same dtype for both inputs and the output. - identity: scalar or Mapping + identity : scalar or Mapping The identity of the monoid such that ``op(x, identity) == x`` for any x. ``identity`` may also be a mapping from dtype to scalar. is_idempotent : bool, default False diff --git a/graphblas/core/operator/semiring.py b/graphblas/core/operator/semiring.py index 32e20d0bf..b333259e3 100644 --- a/graphblas/core/operator/semiring.py +++ b/graphblas/core/operator/semiring.py @@ -175,7 +175,7 @@ class Semiring(OpBase): and the Monoid taking the place of the standard addition operator. Built-in and registered Semirings are located in the ``graphblas.semiring`` namespace - as well as in the ``graphblas.ops`` combined namespace. + as well as in the ``graphblas.op`` combined namespace. """ __slots__ = "_monoid", "_binaryop" diff --git a/graphblas/core/operator/unary.py b/graphblas/core/operator/unary.py index 64f68eb48..fc9c92661 100644 --- a/graphblas/core/operator/unary.py +++ b/graphblas/core/operator/unary.py @@ -133,7 +133,7 @@ class UnaryOp(OpBase): """Takes one input and returns one output, possibly of a different data type. Built-in and registered UnaryOps are located in the ``graphblas.unary`` namespace - as well as in the ``graphblas.ops`` combined namespace. + as well as in the ``graphblas.op`` combined namespace. """ __slots__ = "orig_func", "is_positional", "_is_udt", "_numba_func" diff --git a/graphblas/core/scalar.py b/graphblas/core/scalar.py index 9755acb3c..ce191f374 100644 --- a/graphblas/core/scalar.py +++ b/graphblas/core/scalar.py @@ -182,7 +182,7 @@ def __sizeof__(self): size = ffi_new("size_t*") check_status(lib.GxB_Scalar_memoryUsage(size, self.gb_obj[0]), self) return base + size[0] - raise TypeError("Unable to get size of GrB_Scalar with backend: {backend}") + raise TypeError(f"Unable to get size of GrB_Scalar with backend: {backend}") def isequal(self, other, *, check_dtype=False): """Check for exact equality (including whether the value is missing). @@ -191,7 +191,7 @@ def isequal(self, other, *, check_dtype=False): ---------- other : Scalar Scalar to compare against - check_dtypes : bool, default=False + check_dtype : bool, default=False If True, also checks that dtypes match Returns diff --git a/graphblas/core/ss/binary.py b/graphblas/core/ss/binary.py index d53608818..87c213d32 100644 --- a/graphblas/core/ss/binary.py +++ b/graphblas/core/ss/binary.py @@ -106,20 +106,23 @@ def register_new(name, jit_c_definition, left_type, right_type, ret_type): else: # We use `is_udt=True` to make dtype handling flexible and explicit. rv = BinaryOp(name, is_udt=True) + from .jit_config import _compiling_c_source + gb_obj = ffi_new("GrB_BinaryOp*") - check_status_carg( - lib.GxB_BinaryOp_new( - gb_obj, - NULL, - ret_type._carg, - left_type._carg, - right_type._carg, - ffi_new("char[]", funcname.encode()), - ffi_new("char[]", jit_c_definition.encode()), - ), - "BinaryOp", - gb_obj[0], - ) + with _compiling_c_source(f"gb.binary.{name}"): + check_status_carg( + lib.GxB_BinaryOp_new( + gb_obj, + NULL, + ret_type._carg, + left_type._carg, + right_type._carg, + ffi_new("char[]", funcname.encode()), + ffi_new("char[]", jit_c_definition.encode()), + ), + "BinaryOp", + gb_obj[0], + ) op = TypedJitBinaryOp( rv, funcname, left_type, ret_type, gb_obj[0], jit_c_definition, dtype2=right_type ) diff --git a/graphblas/core/ss/descriptor.py b/graphblas/core/ss/descriptor.py index 20fc126cf..c8a2fcade 100644 --- a/graphblas/core/ss/descriptor.py +++ b/graphblas/core/ss/descriptor.py @@ -153,7 +153,7 @@ def get_descriptor(**opts): Whether and how to compress the data for serialization. The default is "zstd" with ``compression_level=1`` compression_level : int - 1-9 for "lz4hc" compression: (1) is the fastest, (9) is the most compact (default 1). + 1-9 for "lz4hc" compression: (1) is the fastest, (9) is the most compact (default 9). 1-19 for "zstd" compression: (1) is the fastest, (19) is the most compact (default 1). Ignored for other compression methods. @@ -212,7 +212,7 @@ def _set_compression(desc, compression, level): comp = _str_to_compression[compression] if level is not None: if compression not in {"lz4hc", "zstd"}: - raise TypeError('level argument is only valid when using "lz4hc" compression') + raise TypeError('level argument is only valid when using "lz4hc" or "zstd" compression') level = int(level) upper = 9 if compression == "lz4hc" else 19 default = 9 if compression == "lz4hc" else 1 diff --git a/graphblas/core/ss/dtypes.py b/graphblas/core/ss/dtypes.py index eeca52d0b..12f74fbce 100644 --- a/graphblas/core/ss/dtypes.py +++ b/graphblas/core/ss/dtypes.py @@ -31,9 +31,13 @@ def register_new(name, jit_c_definition, *, np_type=None): raise ValueError(f"`name` argument must be a valid Python identifier; got: {name!r}") if name in core.dtypes._registry or hasattr(dtypes.ss, name): raise ValueError(f"{name!r} name for dtype is unavailable") - if len(name) > lib.GxB_MAX_NAME_LEN: + # GxB_MAX_NAME_LEN counts the terminating NUL, so SuiteSparse keeps one fewer + # character; a name that long would no longer match its typedef and would + # fail to compile. + if len(name) >= lib.GxB_MAX_NAME_LEN: raise ValueError( - f"`name` argument is too large. Max size is {lib.GxB_MAX_NAME_LEN}; got {len(name)}" + "`name` argument is too large. " + f"Max size is {lib.GxB_MAX_NAME_LEN - 1}; got {len(name)}" ) if name not in jit_c_definition: raise ValueError("`name` argument must be same name as the typedef in `jit_c_definition`") @@ -42,15 +46,17 @@ def register_new(name, jit_c_definition, *, np_type=None): # Registering a type by its C typedef is a request to compile C, so it is # a fair place to raise SuiteSparse's non-compiling default. - from .jit_config import _enable_jit_for_udt - - _enable_jit_for_udt() + from .jit_config import _compiling_c_source gb_obj = ffi.new("GrB_Type*") - status = lib.GxB_Type_new( - gb_obj, 0, ffi_new("char[]", name.encode()), ffi_new("char[]", jit_c_definition.encode()) - ) - check_status_carg(status, "Type", gb_obj[0]) + with _compiling_c_source(f"gb.dtypes.ss.{name}"): + status = lib.GxB_Type_new( + gb_obj, + 0, + ffi_new("char[]", name.encode()), + ffi_new("char[]", jit_c_definition.encode()), + ) + check_status_carg(status, "Type", gb_obj[0]) # Let SuiteSparse:GraphBLAS determine the size (we gave 0 as size above) size_ptr = ffi_new("size_t*") diff --git a/graphblas/core/ss/indexbinary.py b/graphblas/core/ss/indexbinary.py index 89a38b3fc..85b3253e0 100644 --- a/graphblas/core/ss/indexbinary.py +++ b/graphblas/core/ss/indexbinary.py @@ -111,21 +111,24 @@ def register_new(name, jit_c_definition, x_type, y_type, theta_type, ret_type): ) else: rv = IndexBinaryOp(name, is_udt=True) + from .jit_config import _compiling_c_source + gb_obj = ffi_new("GxB_IndexBinaryOp*") - check_status_carg( - lib.GxB_IndexBinaryOp_new( - gb_obj, - NULL, - ret_type._carg, - x_type._carg, - y_type._carg, - theta_type._carg, - ffi_new("char[]", funcname.encode()), - ffi_new("char[]", jit_c_definition.encode()), - ), - "IndexBinaryOp", - gb_obj[0], - ) + with _compiling_c_source(f"gb.indexbinary.{name}"): + check_status_carg( + lib.GxB_IndexBinaryOp_new( + gb_obj, + NULL, + ret_type._carg, + x_type._carg, + y_type._carg, + theta_type._carg, + ffi_new("char[]", funcname.encode()), + ffi_new("char[]", jit_c_definition.encode()), + ), + "IndexBinaryOp", + gb_obj[0], + ) op = TypedJitIndexBinaryOp( rv, funcname, x_type, ret_type, gb_obj[0], jit_c_definition, dtype2=theta_type ) diff --git a/graphblas/core/ss/indexunary.py b/graphblas/core/ss/indexunary.py index b60837acf..fa394eb4f 100644 --- a/graphblas/core/ss/indexunary.py +++ b/graphblas/core/ss/indexunary.py @@ -105,20 +105,23 @@ def register_new(name, jit_c_definition, input_type, thunk_type, ret_type): else: # We use `is_udt=True` to make dtype handling flexible and explicit. rv = IndexUnaryOp(name, is_udt=True) + from .jit_config import _compiling_c_source + gb_obj = ffi_new("GrB_IndexUnaryOp*") - check_status_carg( - lib.GxB_IndexUnaryOp_new( - gb_obj, - NULL, - ret_type._carg, - input_type._carg, - thunk_type._carg, - ffi_new("char[]", funcname.encode()), - ffi_new("char[]", jit_c_definition.encode()), - ), - "IndexUnaryOp", - gb_obj[0], - ) + with _compiling_c_source(f"gb.indexunary.{name}"): + check_status_carg( + lib.GxB_IndexUnaryOp_new( + gb_obj, + NULL, + ret_type._carg, + input_type._carg, + thunk_type._carg, + ffi_new("char[]", funcname.encode()), + ffi_new("char[]", jit_c_definition.encode()), + ), + "IndexUnaryOp", + gb_obj[0], + ) op = TypedJitIndexUnaryOp( rv, funcname, input_type, ret_type, gb_obj[0], jit_c_definition, dtype2=thunk_type ) diff --git a/graphblas/core/ss/jit_config.py b/graphblas/core/ss/jit_config.py index 8cdc19e25..8f3aa4a9a 100644 --- a/graphblas/core/ss/jit_config.py +++ b/graphblas/core/ss/jit_config.py @@ -20,6 +20,7 @@ warning at import or at first JIT use. """ +import contextlib import os import pathlib import platform @@ -241,7 +242,7 @@ def _auto_fix_jit_at_import(): submodule, and that must not change what any later operation computes or which kernels SuiteSparse is willing to load from its on-disk cache. Compilation is enabled later, by :func:`_enable_jit_for_udt`, when a UDT - actually needs a kernel built. + actually needs a kernel built, or a type or op is registered from C source. """ cfg = _ss_config() if "jit_c_control" not in cfg: @@ -250,7 +251,7 @@ def _auto_fix_jit_at_import(): _repair_jit_compiler(cfg) -# Tri-state: ``None`` until a UDT first asks for a JIT kernel while the +# Tri-state: ``None`` until the first request to compile C source while the # control allows compiling, then the answer to "can this process # JIT-compile?" until ``fix_jit_config`` re-arms it. _jit_enabled_for_udt = None @@ -263,17 +264,19 @@ def _auto_fix_jit_at_import(): def _enable_jit_for_udt(): - """Enable JIT compilation the first time a UDT needs a kernel built. + """Enable JIT compilation the first time C source needs compiling. SuiteSparse defaults ``jit_c_control`` to ``'run'``, which runs kernels already loaded but neither compiles nor loads any. UDT auto-lift wants ``'on'``; without it every UDT op falls back to the Numba - function-pointer path (typically 2-3x slower for elementwise ops). + function-pointer path (typically 2-3x slower for elementwise ops). An op + or type registered from C source has no fallback: under ``'run'`` its + creation fails outright. This is where that bump belongs, rather than at import: registering a - type from its C typedef or arming an op with C source is an act that - plainly involves compiling C, so enabling the compiler is not a surprise. - Reading ``gb.ss.about`` is not, so it leaves the setting alone. + type or an op from C source, or arming an op with C source, is an act + that plainly involves compiling C, so enabling the compiler is not a + surprise. Reading ``gb.ss.about`` is not, so it leaves the setting alone. Only the default ``'run'`` is raised. An explicit ``'off'``, ``'pause'``, or ``'load'`` is honored, and ``'load'`` is also where SuiteSparse leaves @@ -314,6 +317,67 @@ def _enable_jit_for_udt(): return _jit_enabled_for_udt +@contextlib.contextmanager +def _compiling_c_source(name): + """Enable the JIT to create ``name`` from C source, and explain a failure. + + SuiteSparse compiles the C source, or loads the kernel from its cache, as + part of creating the type or op. When it cannot, it reports a bare + ``NullPointer`` (op) or ``InvalidValue`` (type), or from 9.4 a + ``JitError`` for a failed compile, none with a message. ``jit_c_control`` + before and after says which case it was, so the same exception type is + raised again with the reason, chained to the original. + """ + from ...exceptions import InvalidValue, JitError, NullPointer + + cfg = _ss_config() + initial = cfg.get("jit_c_control") + _enable_jit_for_udt() + before = cfg.get("jit_c_control") + try: + yield + except (NullPointer, InvalidValue, JitError) as exc: + after = cfg.get("jit_c_control") + if after in {"off", "pause"} and after == before: + reason = ( + f"jit_c_control is {after!r}, so SuiteSparse will not compile it. " + "Set gb.ss.config['jit_c_control'] = 'on' to allow that." + ) + elif after == "load" and before == "on": + reason = ( + "SuiteSparse failed to compile it and set jit_c_control to 'load', so " + "nothing else compiles in this process until jit_c_control is set back " + "to 'on'. SuiteSparse prints the compiler's output when " + "gb.ss.config['burble'] is True." + ) + elif after == before == "load" and initial != "load": + # ``_enable_jit_for_udt`` only ever writes 'on', so 'load' here means + # the probe it just ran failed to compile. + reason = ( + "the C compiler does not work here: SuiteSparse could not compile a " + "small test type, and set jit_c_control to 'load'. To see the " + "compiler's output, set gb.ss.config['burble'] = True and " + "jit_c_control back to 'on', then try again." + ) + elif after == before == "load": + reason = ( + "jit_c_control is 'load', so SuiteSparse only loads kernels already in " + "its cache, and this one is not there. SuiteSparse also sets 'load' " + "itself after a compile fails. Set gb.ss.config['jit_c_control'] = " + "'on' to compile it." + ) + elif after == before == "run": + reason = ( + "jit_c_control is still 'run', which neither compiles nor loads: " + "python-graphblas did not find the configured C compiler, or " + "SuiteSparse was built without the JIT. gb.ss.fix_jit_config() looks " + "for a compiler and turns the JIT on." + ) + else: + raise + raise type(exc)(f"Cannot create {name} from C source: {reason}") from exc + + # Keyed by ``(op_name, dtype_name)`` so each distinct pair warns once. # A user who registers several UDTs gets one warning per (op, dtype) pair # rather than a single global swallow. diff --git a/graphblas/core/ss/matrix.py b/graphblas/core/ss/matrix.py index 0580b7581..2c9a4d99e 100644 --- a/graphblas/core/ss/matrix.py +++ b/graphblas/core/ss/matrix.py @@ -4059,10 +4059,10 @@ def serialize(self, compression="default", level=None, **opts): - "lz4hc": LZ4 compression that allows the compression level (1-9) to be set. Low compression level (1) is faster, high (9) is more compact. Default is 9. - "zstd": ZSTD compression, which allows compression level (1-19) to be set. - Low compression level (1) is faster, high (19) is more compact. Default is 19. + Low compression level (1) is faster, high (19) is more compact. Default is 1. - "none" or None: no compression - level : int [1-9], optional - The compression level, between 1 to 9, to use with "lz4hc" and "zstd" compression. + level : int, optional + The compression level to use with "lz4hc" (1-9) and "zstd" (1-19) compression. Level 1 is the fastest and largest, and is the default for "zstd" compression. Level 9 is the default when using "lz4hc" compression. nthreads : int, optional diff --git a/graphblas/core/ss/unary.py b/graphblas/core/ss/unary.py index 0b7ced3c8..e0be45db9 100644 --- a/graphblas/core/ss/unary.py +++ b/graphblas/core/ss/unary.py @@ -90,19 +90,22 @@ def register_new(name, jit_c_definition, input_type, ret_type): else: # We use `is_udt=True` to make dtype handling flexible and explicit. rv = UnaryOp(name, is_udt=True) + from .jit_config import _compiling_c_source + gb_obj = ffi_new("GrB_UnaryOp*") - check_status_carg( - lib.GxB_UnaryOp_new( - gb_obj, - NULL, - ret_type._carg, - input_type._carg, - ffi_new("char[]", funcname.encode()), - ffi_new("char[]", jit_c_definition.encode()), - ), - "UnaryOp", - gb_obj[0], - ) + with _compiling_c_source(f"gb.unary.{name}"): + check_status_carg( + lib.GxB_UnaryOp_new( + gb_obj, + NULL, + ret_type._carg, + input_type._carg, + ffi_new("char[]", funcname.encode()), + ffi_new("char[]", jit_c_definition.encode()), + ), + "UnaryOp", + gb_obj[0], + ) op = TypedJitUnaryOp(rv, funcname, input_type, ret_type, gb_obj[0], jit_c_definition) rv._add(op, is_jit=True) setattr(module, funcname, rv) diff --git a/graphblas/core/ss/vector.py b/graphblas/core/ss/vector.py index 6c79a338b..80932da73 100644 --- a/graphblas/core/ss/vector.py +++ b/graphblas/core/ss/vector.py @@ -1630,10 +1630,10 @@ def serialize(self, compression="default", level=None, **opts): - "lz4hc": LZ4 compression that allows the compression level (1-9) to be set. Low compression level (1) is faster, high (9) is more compact. Default is 9. - "zstd": ZSTD compression, which allows compression level (1-19) to be set. - Low compression level (1) is faster, high (19) is more compact. Default is 19. + Low compression level (1) is faster, high (19) is more compact. Default is 1. - "none" or None: no compression - level : int [1-9], optional - The compression level, between 1 to 9, to use with "lz4hc" and "zstd" compression. + level : int, optional + The compression level to use with "lz4hc" (1-9) and "zstd" (1-19) compression. Level 1 is the fastest and largest, and is the default for "zstd" compression. Level 9 is the default when using "lz4hc" compression. nthreads : int, optional diff --git a/graphblas/core/utils.py b/graphblas/core/utils.py index aef2f76a3..72f7219bd 100644 --- a/graphblas/core/utils.py +++ b/graphblas/core/utils.py @@ -153,7 +153,7 @@ def get_shape(nrows, ncols, dtype=None, **arrays): break else: raise ValueError( - "Either nrows and ncols must be provided, or one of the following arrays" + "Either nrows and ncols must be provided, or one of the following arrays " f'must be 2d (from which to get nrows and ncols): {", ".join(arrays)}' ) if nrows is None: diff --git a/graphblas/core/vector.py b/graphblas/core/vector.py index 8c73ecc48..5d8c8fef9 100644 --- a/graphblas/core/vector.py +++ b/graphblas/core/vector.py @@ -335,7 +335,7 @@ def __sizeof__(self): size = ffi_new("size_t*") check_status(lib.GxB_Vector_memoryUsage(size, self.gb_obj[0]), self) return size[0] + object.__sizeof__(self) - raise TypeError("Unable to get size of Vector with backend: {backend}") + raise TypeError(f"Unable to get size of Vector with backend: {backend}") def isequal(self, other, *, check_dtype=False, **opts): """Check for exact equality (same size, same structure). @@ -344,7 +344,7 @@ def isequal(self, other, *, check_dtype=False, **opts): ---------- other : Vector The vector to compare against - check_dtypes : bool, default=False + check_dtype : bool, default=False If True, also checks that dtypes match Returns @@ -469,7 +469,7 @@ def to_coo(self, dtype=None, *, indices=True, values=True, sort=True): ---------- dtype : Requested dtype for the output values array. - indices :bool, default=True + indices : bool, default=True Whether to return indices; will return ``None`` for indices if ``False`` values : bool, default=True Whether to return values; will return ``None`` for values if ``False`` @@ -609,8 +609,6 @@ def diag(self, k=0, *, name=None): ---------- k : int Off-diagonal offset in the Matrix. - dtype : - Data type of the new Matrix. Normal typecasting rules apply. name : str, optional Name to give the new Matrix. @@ -800,10 +798,8 @@ def from_scalar(cls, value, size, dtype=None, *, name=None, **opts): ---------- value : scalar Scalar value used to fill the Vector. - nrows : int - Number of rows. - ncols : int - Number of columns. + size : int + Size of the Vector. dtype : DataType, optional Data type of the Vector. If not provided, the scalar value will be inspected to choose an appropriate dtype. @@ -1316,7 +1312,7 @@ def vxm(self, other, op=semiring.plus_times): Parameters ---------- - other: Matrix + other : Matrix The matrix on the right side in the computation op : :class:`~graphblas.core.operator.Semiring` Semiring used in the computation diff --git a/graphblas/io/_awkward.py b/graphblas/io/_awkward.py index fd4a9bab8..7b9f18906 100644 --- a/graphblas/io/_awkward.py +++ b/graphblas/io/_awkward.py @@ -14,7 +14,7 @@ def to_awkward(A, format=None): ---------- A : Matrix or Vector GraphBLAS object to be converted - format : str {'csr', 'csc', 'hypercsr', 'hypercsc', 'vec} + format : str {'csr', 'csc', 'hypercsr', 'hypercsc', 'vec'} Default format is csr for Matrix; vec for Vector The Awkward Array will have top-level attributes based on format: diff --git a/graphblas/io/_matrixmarket.py b/graphblas/io/_matrixmarket.py index cf425e2e9..4d7fcca22 100644 --- a/graphblas/io/_matrixmarket.py +++ b/graphblas/io/_matrixmarket.py @@ -21,7 +21,7 @@ def mmread(source, engine="auto", *, dup_op=None, name=None, **kwargs): Parameters ---------- source : str or file - Filename (.mtx or .mtz.gz) or file-like object + Filename (.mtx or .mtx.gz) or file-like object engine : {"auto", "scipy", "fmm", "fast_matrix_market"}, default "auto" How to read the matrix market file. "scipy" uses ``scipy.io.mmread``, "fmm" and "fast_matrix_market" uses ``fast_matrix_market.mmread``, diff --git a/graphblas/ss/_core.py b/graphblas/ss/_core.py index 91e1496e9..5e851b7cc 100644 --- a/graphblas/ss/_core.py +++ b/graphblas/ss/_core.py @@ -136,7 +136,7 @@ class GlobalConfig(BaseConfig): Which GPU to use; default is -1, which means do not run on the GPU. Only available for SuiteSparse:GraphBLAS >=8 **GPU support is a work in progress; not recommended to use** - jit_c_control : {"off", "pause", "run", "load", "on} + jit_c_control : {"off", "pause", "run", "load", "on"} Control the CPU JIT: "off" : do not use the JIT and free all JIT kernels if loaded "pause" : do not run JIT kernels, but keep any loaded diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index 34e4767d6..eeeeac206 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -3,6 +3,7 @@ import subprocess import sys import sysconfig +import uuid from pathlib import Path import numpy as np @@ -195,6 +196,149 @@ def test_fix_jit_config_first_call_agrees_with_the_second(): assert lines["first"] == lines["second"], report +# Run in a subprocess: this process has already probed the JIT and cached the +# answer, and a fresh one has to probe at its first registration. Each +# registration starts from ``'run'``, so one registrar raising the control +# cannot cover for another that does not. +_SS_REGISTER_NEW_PROBE = """ +import graphblas as gb +from graphblas import binary, indexbinary, indexunary, select, unary +from graphblas.core.operator.indexbinary import _has_idxbinop + +print("package: " + gb.__file__) +registrars = { + "unary": lambda: unary.ss.register_new( + "rn_neg", "void rn_neg (double *z, double *x) { (*z) = -(*x) ; }", "FP64", "FP64" + ), + "binary": lambda: binary.ss.register_new( + "rn_absdiff", + "void rn_absdiff (double *z, double *x, double *y) { (*z) = fabs ((*x) - (*y)) ; }", + "FP64", "FP64", "FP64", + ), + "indexunary": lambda: indexunary.ss.register_new( + "rn_diffy", + "void rn_diffy (double *z, double *x, GrB_Index i, GrB_Index j, double *y) " + "{ (*z) = (i + j) * fabs ((*x) - (*y)) ; }", + "FP64", "FP64", "FP64", + ), + "select": lambda: select.ss.register_new( + "rn_woot", + "void rn_woot (bool *z, const int32_t *x, GrB_Index i, GrB_Index j, int32_t *y) " + "{ (*z) = ((*x) + i + j == (*y)) ; }", + "INT32", "INT32", + ), +} +if _has_idxbinop: + registrars["indexbinary"] = lambda: indexbinary.ss.register_new( + "rn_add_theta", + "void rn_add_theta (double *z, double *x, GrB_Index ix, GrB_Index jx, " + "double *y, GrB_Index iy, GrB_Index jy, double *theta) " + "{ (*z) = (*x) + (*y) + (*theta) ; }", + "FP64", "FP64", "FP64", "FP64", + ) +for kind, register in registrars.items(): + gb.ss.config["jit_c_control"] = "run" + try: + register() + except Exception as exc: + print(kind + ": " + type(exc).__name__) + else: + print(kind + ": ok") +""" + + +@pytest.mark.skipif("_IS_SSGB7") +def test_ss_register_new_enables_the_jit(): + """Registering an op from C source raises SuiteSparse's default ``'run'``. + + SuiteSparse compiles the C definition, or loads it from its cache, when the + op is created. ``'run'`` does neither, so without the raise the + registration fails. Each registrar has to ask for the JIT itself: since + ``gb.ss`` stopped raising the control at import, nothing else does it + first. + """ + if not _JIT_WORKS_AT_IMPORT: + pytest.skip("JIT not compiled in or not working here; nothing can compile") + lines, report = _run_in_fresh_process(_SS_REGISTER_NEW_PROBE) + kinds = ["unary", "binary", "indexunary", "select"] + kinds += ["indexbinary"] if _has_idxbinop else [] + assert {kind: lines.get(kind) for kind in kinds} == dict.fromkeys(kinds, "ok"), report + + +@pytest.mark.skipif("_IS_SSGB7") +def test_ss_register_new_says_why_the_jit_cannot_build_it(monkeypatch): + """A C-source registration the JIT cannot build names the reason. + + For each of these SuiteSparse reports a bare ``NullPointer`` (op) or + ``InvalidValue`` (type) with no message. None of them compiles anything, so + this runs with or without a working compiler; a library built without the + JIT skips ``'load'`` and the failed probe. + """ + from graphblas.core.ss import jit_config + from graphblas.exceptions import InvalidValue, NullPointer + + # New names every run, so no cached kernel can satisfy ``'load'``. + suffix = uuid.uuid4().hex[:8] + op_name = f"why_neg_{suffix}" + cdef = f"void {op_name} (double *z, double *x) {{ (*z) = -(*x) ; }}" + type_name = f"why_t_{suffix}" + tdef = f"typedef struct {{ int64_t why_a ; }} {type_name} ;" + prev = gb.ss.config["jit_c_control"] + try: + for control, match in [ + ("off", "jit_c_control is 'off', so SuiteSparse will not compile it"), + ("load", "jit_c_control is 'load', so SuiteSparse only loads kernels"), + ]: + gb.ss.config["jit_c_control"] = control + if gb.ss.config["jit_c_control"] != control: + # A library built without the JIT clamps 'load' down to 'run'. + continue + with pytest.raises(NullPointer, match=match): + unary.ss.register_new(op_name, cdef, "FP64", "FP64") + with pytest.raises(InvalidValue, match=match): + dtypes.ss.register_new(type_name, tdef) + # A process that cannot compile leaves SuiteSparse's default in place. + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", False) + gb.ss.config["jit_c_control"] = "run" + with pytest.raises(NullPointer, match="jit_c_control is still 'run'"): + unary.ss.register_new(op_name, cdef, "FP64", "FP64") + # A probe that fails in this very call leaves 'load' before the op is created. + gb.ss.config["jit_c_control"] = "on" + if gb.ss.config["jit_c_control"] == "on": + + def failed_probe(cfg): + cfg["jit_c_control"] = "load" + return False + + monkeypatch.setattr(jit_config, "_jit_enabled_for_udt", None) + monkeypatch.setattr(jit_config, "jit_compiler_is_usable", lambda: True) + monkeypatch.setattr(jit_config, "_probe_jit", failed_probe) + gb.ss.config["jit_c_control"] = "run" + with pytest.raises(NullPointer, match="the C compiler does not work here"): + unary.ss.register_new(op_name, cdef, "FP64", "FP64") + finally: + gb.ss.config["jit_c_control"] = prev + + +@pytest.mark.skipif("_IS_SSGB7") +def test_ss_register_new_says_the_compile_failed(): + """C source that does not compile says so, and that ``jit_c_control`` is now ``'load'``.""" + if not _JIT_WORKS_AT_IMPORT: + pytest.skip("JIT not compiled in or not working here; nothing compiles to fail") + from graphblas.exceptions import JitError, NullPointer + + cdef = "void why_bad (double *z, double *x) { (*z) = why_undeclared ; }" + prev = gb.ss.config["jit_c_control"] + try: + gb.ss.config["jit_c_control"] = "on" + # SuiteSparse reports a failed compile as JitError from 9.4 on. + with pytest.raises((NullPointer, JitError), match="SuiteSparse failed to compile it"): + unary.ss.register_new("why_bad", cdef, "FP64", "FP64") + assert gb.ss.config["jit_c_control"] == "load" + finally: + gb.ss.config["jit_c_control"] = prev + + @pytest.mark.skipif("_IS_SSGB7") def test_auto_fix_jit_at_import_left_compiler_usable(): """The import-time repair leaves a compiler the UDT path can actually use.""" @@ -286,8 +430,11 @@ def test_enable_jit_for_udt_re_raises_the_live_control(): jit_config._jit_enabled_for_udt = prev_enabled -@pytest.mark.skipif("_IS_SSGB7") @pytest.mark.skipif("not supports_udfs") +# The op this test arms, binary.plus on a UDT, carries C source only on SS 9+ +# (see test_udt_jit_c_source_introspection). On 8.x the stand-in config tests +# above still check that _enable_jit_for_udt leaves 'load' alone. +@pytest.mark.skipif("not _has_jit_set") def test_failed_compile_is_not_retried_at_the_next_arming(): """A compile that fails is tried once, not again at every later UDT op. @@ -444,6 +591,21 @@ def test_jit_udt(): assert dtype.np_type == np_type +@pytest.mark.skipif("_IS_SSGB7") +def test_jit_udt_name_leaves_room_for_the_nul(): + """A type name as long as ``GxB_MAX_NAME_LEN`` is rejected before SuiteSparse sees it. + + SuiteSparse keeps one character fewer, so that name would no longer match + its typedef, and the registration would fail as a compile error. + """ + from graphblas.core import lib + + limit = lib.GxB_MAX_NAME_LEN + name = "n" * limit + with pytest.raises(ValueError, match=f"Max size is {limit - 1}; got {limit}"): + dtypes.ss.register_new(name, f"typedef struct {{ int64_t a ; }} {name} ;") + + def test_jit_unary(v): cdef = "void square (float *z, float *x) { (*z) = (*x) * (*x) ; } ;" if _IS_SSGB7: diff --git a/pyproject.toml b/pyproject.toml index cdf8867c8..9bd988a10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,14 +85,9 @@ io = [ "python-graphblas[networkx,scipy]", "python-graphblas[numba]; python_version<'3.15'", # make optional where numba is not supported "awkward >=2.0", - # sparse needs numba, so it stays optional. numba gained py3.13 support in 0.61 and - # py3.14 in 0.63; sparse 0.16 is the first release that postdates py3.13, and 0.18 the - # first that postdates py3.14. - "sparse >=0.14; python_version<'3.13'", - "sparse >=0.16; python_version=='3.13'", - # Upper guard for the same reason as numba above: sparse depends on numba, so it has - # to drop out wherever numba does, not just trail it. - "sparse >=0.18; python_version>='3.14' and python_version<'3.15'", + # sparse needs numba, so it stays optional and takes numba's upper guard from above: + # it has to drop out wherever numba does. + "sparse >=0.14; python_version<'3.15'", # fast-matrix-market is no longer maintained; last supported Python is 3.12 ] viz = ["python-graphblas[networkx,scipy]", "matplotlib >=3.6"] diff --git a/scripts/ci_pick_versions.py b/scripts/ci_pick_versions.py index 2fcfafc02..ebd003d96 100755 --- a/scripts/ci_pick_versions.py +++ b/scripts/ci_pick_versions.py @@ -139,16 +139,20 @@ "3.14": ["6.0", ""], } -# sparse is noarch, so there are no per-Python builds to check; the split is by what -# each release plausibly saw. The 0.14/0.15 series began well before py3.13 (0.15.0 is -# Jan 2024), and 0.16/0.17 (Apr/May 2025) predate py3.14. To skip sparse on a Python, -# give it a ["NA"] pool: a bare "NA" would be indexed as a string by `random.choice`. -# MAINT: 2026-09-16 sparse 0.19.2 is latest; 0.18 verified against py3.14 locally +# sparse is noarch and caps neither Python nor numpy: every release here declares numba +# >=0.49, numpy >=1.17 and a Python floor at or below our oldest (>=3.8 for 0.14/0.15, +# >=3.10 for 0.16/0.17, >=3.11 for 0.18/0.19), so every pin installs on every Python and +# the rows are all the same. It depends on numba, so the workflow drops sparse wherever it +# skips numba. To skip sparse on a Python, give it a ["NA"] pool: a bare "NA" would be +# indexed as a string by `random.choice`. +# MAINT: 2026-09-26 sparse 0.19.2 is latest. Each pin, as conda resolves it, passes the +# pydata-sparse tests on py3.13 and py3.14 with the oldest and newest numpy/numba pair +# each one draws. SPARSE_VERSIONS = { "3.11": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], "3.12": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], - "3.13": ["0.16", "0.17", "0.18", "0.19", ""], - "3.14": ["0.18", "0.19", ""], + "3.13": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], + "3.14": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], } # PSG versions to pair with numpy 1.x (only reachable on py3.11/py3.12, since @@ -424,6 +428,10 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): pool = [n for n in numba_pool if n and NUMBA_MAX_NUMPY[n] > npver and _ver(n) >= min_ver] v["numba"] = random.choice(pool) if pool else "NA" + # --- sparse --- + # No rules needed: sparse caps neither Python nor numpy, and the workflow drops it + # whenever numba is skipped or NA. + # --------------------------------------------------------------------------- # Main logic