diff --git a/docs/user_guide/types.rst b/docs/user_guide/types.rst index 36d93e8a4..65b941cf9 100644 --- a/docs/user_guide/types.rst +++ b/docs/user_guide/types.rst @@ -48,7 +48,14 @@ follow the same rule, field by field (see :doc:`udt`). A comparison has no result type to keep, so an integer outside the operand's range compares exactly, as in numpy: ``v < 300`` is True for every element, as is ``w < 2**63`` for an INT64 vector ``w``. -Nor does ``v / 300`` raise, since its result is FP64 whatever the integer. An +Nor does an operator whose result has another type raise: ``v / 300`` is FP64 +whatever the integer, and ``v.apply(binary.isclose, right=300)`` is False for +every element. A user-defined operator whose result has another type gets the +integer exactly where the operand's type and the integer's share an integer +type (INT16 for ``300`` here), and raises ``OverflowError`` where they would +share only a floating-point type, which can round the integer: beside an INT64 +vector, ``2**63`` would meet the elements in FP64, where ``2**63 - 1`` and +``2**63`` are equal. An FP32 vector ``== 0.1`` compares in FP32, and ``== 1e300`` compares with infinity, with a ``RuntimeWarning`` as in numpy. The defaults of ``ewise_union`` follow the rule, and so does the ``fill_value`` of ``to_dense``, except that a diff --git a/docs/user_guide/udt.rst b/docs/user_guide/udt.rst index 44af6a1f0..96d8e1f91 100644 --- a/docs/user_guide/udt.rst +++ b/docs/user_guide/udt.rst @@ -74,10 +74,19 @@ Field type rules: struct) are supported as fields, and built-in operators lift through them. In the dict and dataclass forms, write an array field as ``"FP64[3]"``; the dict form also takes an existing UDT, such as ``{"id": int, "pos": point3}``. -- Numba cannot type an array of ``bool``, ``datetime64`` or ``timedelta64``, so - with Numba installed, registering a UDT that holds one raises ``ValueError``. - Use an array of ``uint8`` for ``bool``, and of ``int64`` for the others. -- Strings and Python objects are not supported. +- Numba cannot type an array of ``bool``, ``datetime64``, ``timedelta64``, + ``str`` or ``bytes``, so with Numba installed, registering a UDT that holds + one raises ``ValueError``. Use an array of ``uint8`` for ``bool`` and + ``bytes`` (one per byte), of ``uint32`` for ``str`` (one per character), and + of ``int64`` for the others. +- Text (``str`` or ``bytes``, such as ``np.dtype("S5")``), ``datetime64`` and + ``timedelta64`` may be fields, or a whole UDT that is neither record nor + array. Such a UDT can be stored and extracted. With Numba, it can also be + compared (``==``, ``!=`` and ``isequal``), and ``first``, ``second``, ``pair`` + and ``any`` work on it, but arithmetic operators do not lift to it: ``plus`` + and ``min`` raise ``KeyError``. Without Numba, only a ``Scalar``'s ``==``, + ``!=`` and ``isequal``, and ``pair``, work on it. +- Python objects (``np.dtype(object)``) are not supported. - Dataclass annotation strings (e.g., ``"int"``) resolve through ``lookup_dtype``. diff --git a/graphblas/core/base.py b/graphblas/core/base.py index e8940ad9b..a3562f922 100644 --- a/graphblas/core/base.py +++ b/graphblas/core/base.py @@ -365,7 +365,7 @@ def _update(self, expr, mask=None, accum=None, replace=False, input_mask=None, * expr = expr._extract_delayed() elif type(expr) is type(self): # Simple assignment (w << v) - if (cast_op := _store_cast_op(expr.dtype, self.dtype)) is not None: + if (cast_op := _store_cast_op(expr.dtype, self.dtype, is_value=True)) is not None: expr = expr.apply(cast_op) self._update(expr, mask, accum, replace, input_mask, opts=opts) return @@ -407,7 +407,9 @@ def _update(self, expr, mask=None, accum=None, replace=False, input_mask=None, * from .matrix import Matrix, MatrixExpression, TransposedMatrix if type(expr) is TransposedMatrix and type(self) is Matrix: - if (cast_op := _store_cast_op(expr.dtype, self.dtype)) is not None: + if ( + cast_op := _store_cast_op(expr.dtype, self.dtype, is_value=True) + ) is not None: # C << A.T of another UDT type: the cast reads A transposed. expr = expr.apply(cast_op) else: @@ -570,6 +572,7 @@ class BaseExpression: "expr_repr", "dtype", "_value", + "_right_literal", "__weakref__", ) output_type = None @@ -605,6 +608,9 @@ def __init__( self.dtype = op.return_type else: self.dtype = dtype + # The right operand as the caller gave it, a literal a comparison typed + # beside the values; the select helpers compare indices with it instead. + self._right_literal = None self._value = None def new(self, dtype=None, *, mask=None, name=None, **opts): diff --git a/graphblas/core/literals.py b/graphblas/core/literals.py index 810bde22c..d43c97aa8 100644 --- a/graphblas/core/literals.py +++ b/graphblas/core/literals.py @@ -22,10 +22,11 @@ import numpy as np +from .. import _STANDARD_OPERATOR_NAMES from ..dtypes import FP64, lookup_dtype from . import utils from .dtypes import _nats_in_leaf_units, _view_if_same_layout -from .operator import IndexUnaryOp, SelectOp, TypedOpBase, binary_from_string +from .operator import IndexUnaryOp, ParameterizedUdf, SelectOp, TypedOpBase, binary_from_string from .operator.binary import BinaryOp from .operator.monoid import Monoid from .udt import ( @@ -97,7 +98,7 @@ def _literal_operand(dtype, value, op, *, left=False): # by a comparison or by an op that gives another type (truediv). if isinstance(op, str): op = binary_from_string(op) - if op.name in _COMPARISONS: + if op.name in _COMPARISONS and not op._anonymous: return (math.inf if value > 0 else -math.inf), FP64 return value, literal_dtype if getattr(op, "_udt_types", True) is None: @@ -114,7 +115,9 @@ def _literal_operand(dtype, value, op, *, left=False): exact = is_comparison or op.name == "truediv" if is_comparison and not isinstance(value, (np.generic, np.ndarray)): np_type = dtype.np_type - parts = _dict_to_record(np_type, value) if isinstance(value, dict) else value + parts = value + if isinstance(value, dict) and np_type.names is not None: + parts = _dict_to_record(np_type, value) # First: making an int beyond its leaf infinity reshapes the part. _refuse_misfit_fields(dtype, np_type, parts) if _int_beyond_leaf(np_type, parts) is not None: @@ -179,13 +182,15 @@ def _weak_builtin_literal_dtype(dtype, value, op=None): # keep, so an int out of range is typed exactly, as for a comparison: # int8_vec / 300 is FP64, as numpy divides integers in float64. np_type = _weak_literal_leaf([value], kind, dtype.np_type, exact=True) - if np_type == dtype.np_type: - rv = dtype - else: - try: - rv = lookup_dtype(np_type) - except ValueError: # A complex literal where the backend has no complex types - return None + if np.result_type(dtype.np_type, np_type).kind not in "iu" and not _is_builtin_op(op): + # The two then unify to a float type, which rounds an int beyond 64 + # bits: in FP64, the INT64 2**63 - 1 and the int 2**63 are equal. A + # built-in op that gives another type computes in float anyway (a + # comparison is given infinity instead), but a user's op may compare. + raise + # Where the backend has no complex types (vanilla), lookup_dtype registers a + # complex literal's dtype as a UDT, so the literal keeps its own type. + rv = dtype if np_type == dtype.np_type else lookup_dtype(np_type) if kind != "b" and rv.np_type in _SINGLE_PRECISION: parts = (value.real, value.imag) if kind == "c" else (float(value),) try: @@ -204,15 +209,29 @@ def _weak_builtin_literal_dtype(dtype, value, op=None): return rv +def _is_builtin_op(op): + """Whether python-graphblas defines ``op``, or a built-in parameterized op made it.""" + if op._anonymous: + info = getattr(getattr(op, "orig_func", None), "_parameterized_info", None) + return info is not None and _is_builtin_op(info[0]) + module = op._op_class._modname if isinstance(op, ParameterizedUdf) else op._modname + return f"{module}.{op.name}" in _STANDARD_OPERATOR_NAMES + + def _gives_another_type(op, dtype): - """Whether built-in ``op`` on two operands of built-in ``dtype`` gives another type. + """Whether ``op`` on two operands of built-in ``dtype`` gives another type. - ``truediv`` on integers gives FP64, and the numpy ufuncs that compute in - float give a float type, as numpy computes them in a float loop. + ``truediv`` on integers gives FP64, ``isclose`` gives BOOL, and the numpy + ufuncs that compute in float give a float type, as numpy computes them in + a float loop. A user's op counts by the type it gives too. A + parameterized op is typed with its default parameters, as + ``get_typed_op`` types it. """ - if op is None or getattr(op, "_anonymous", True) or not hasattr(op, "__getitem__"): + if op is None: return False try: + if isinstance(op, ParameterizedUdf): + op = op() return op[dtype].return_type != dtype except Exception: # An op that does not take this dtype reports it itself, later. return False @@ -257,7 +276,7 @@ def _literal_type(dtype, value, *, exact=False): if _keeps_value(value, np_type): return dtype return lookup_dtype(np.dtype(f"{np_type.kind}{max(len(value), 1)}")) - if np_type.kind in "mM" and (date := _as_numpy_date(value, np_type.kind)) is not None: + if np_type.kind in "mM" and (date := _as_numpy_date(value, np_type)) is not None: if date.dtype.kind == np_type.kind and _keeps_value(date, np_type): return dtype return lookup_dtype(date.dtype) @@ -611,7 +630,7 @@ def leaf_dtype(leaf, part, *, in_record=False): # takes the field's type, and converting it reports what is wrong. if isinstance(part, (str, bytes)) and base.kind in "SU": own = np.dtype(f"{base.kind}{max(len(part), 1)}") - elif (date := _as_numpy_date(part, base.kind)) is not None: + elif (date := _as_numpy_date(part, base)) is not None: part = date own = date.dtype if own.kind != base.kind: @@ -674,21 +693,27 @@ def record_leaves(np_type, part): return dtype if literal_type == np_type else lookup_dtype(literal_type) -def _as_numpy_date(value, kind): +def _as_numpy_date(value, leaf): """Return ``value`` as numpy's ``datetime64`` or ``timedelta64``, or ``None`` if it is neither. A numpy date or duration is itself; a Python ``datetime.date`` or ``datetime.datetime`` is a date in days or microseconds and a ``datetime.timedelta`` a duration in microseconds, as numpy makes them; text is a date in the unit it is written to (``"2020-01-01T12:00"`` is in - minutes) when ``kind``, the leaf's, is ``"M"`` and numpy can read it. + minutes) when ``leaf``, the numpy dtype it meets, is a date and numpy can + read it. Text numpy reads as NaT (``"NaT"`` in any case, or empty) is NaT + in the leaf's unit: numpy would give it no unit, which numpy 2.5 deprecates. """ if isinstance(value, (np.datetime64, np.timedelta64)): return value try: if isinstance(value, datetime.timedelta): return np.timedelta64(value) - if isinstance(value, datetime.date) or (kind == "M" and isinstance(value, (str, bytes))): + if isinstance(value, datetime.date): + return np.datetime64(value) + if leaf.kind == "M" and isinstance(value, (str, bytes)): + if value.lower() in ("nat", "", b"nat", b""): + return np.datetime64("NaT", np.datetime_data(leaf)[0]) return np.datetime64(value) except ValueError: # text that is not a date; converting it reports that pass diff --git a/graphblas/core/matrix.py b/graphblas/core/matrix.py index 166f263a0..049b39772 100644 --- a/graphblas/core/matrix.py +++ b/graphblas/core/matrix.py @@ -2426,6 +2426,7 @@ def apply(self, op, right=None, *, left=None): """ method_name = "apply" + right_literal = None extra_message = ( "apply only accepts UnaryOp with no scalars or BinaryOp with `left` or `right` scalar " "or IndexUnaryOp with `right` thunk." @@ -2495,6 +2496,7 @@ def apply(self, op, right=None, *, left=None): expr_repr = "{1.name}.apply({op}, left={0._expr_name})" elif left is None: if type(right) is not Scalar: + right_literal = right right, dtype = _literal_operand(self.dtype, right, op) try: right = Scalar.from_value(right, dtype, is_cscalar=None, name="") @@ -2538,7 +2540,7 @@ def apply(self, op, right=None, *, left=None): expr_repr = "{0.name}.apply({op}, right={1._expr_name})" else: raise TypeError("Cannot provide both `left` and `right` to apply") - return MatrixExpression( + expr = MatrixExpression( method_name, cfunc_name, args, @@ -2549,6 +2551,8 @@ def apply(self, op, right=None, *, left=None): at=self._is_transposed, bt=self._is_transposed, ) + expr._right_literal = right_literal + return expr def select(self, op, thunk=None): """Create a new Matrix by applying ``op`` to each element of the Matrix diff --git a/graphblas/core/operator/utils.py b/graphblas/core/operator/utils.py index 37da5cc5f..c3a8f5b8f 100644 --- a/graphblas/core/operator/utils.py +++ b/graphblas/core/operator/utils.py @@ -1,6 +1,7 @@ from types import BuiltinFunctionType, FunctionType, ModuleType from ... import ( + _STANDARD_OPERATOR_NAMES, backend, binary, config, @@ -14,6 +15,7 @@ ) from ...dtypes import UINT64, lookup_dtype, unify from ..expr import InfixExprBase +from ..udt import _monoid_pair_type from .base import ( _SS_OPERATORS, OpBase, @@ -49,6 +51,18 @@ # the tuple-form ``__reduce__`` that re-registers the function in the # child. IndexBinaryOp._initialize() + # Every op the namespaces hold now is python-graphblas's own, also ones made + # outside ``register_new`` (cdiv, truediv, rtruediv, the semirings on cdiv) and + # ones made on first use (binom). Record them all as standard: they pickle by + # name, and ``literals._is_builtin_op`` counts them built-in, with no list to + # keep in step. + for _module in [binary, unary, monoid, semiring, indexunary, select, indexbinary]: + _modname = _module.__name__.rsplit(".", 1)[-1] + _STANDARD_OPERATOR_NAMES.update(f"{_modname}.{key}" for key in _module._delayed) + for _val in list(vars(_module).values()): + if isinstance(_val, (OpBase, ParameterizedUdf)) and not _val._anonymous: + _STANDARD_OPERATOR_NAMES.add(f"{_modname}.{_val.name}") + del _module, _modname, _val except Exception: # pragma: no cover (debug) # Exceptions here can often get ignored by Python import traceback @@ -78,12 +92,14 @@ def get_typed_op(op, dtype, dtype2=None, *, is_left_scalar=False, is_right_scala return op[UINT64] if op._udt_types is None: raise KeyError(f"{op.name} does not work with ({dtype}, {dtype2})") - if kind == "binary" and isinstance(op, Monoid): + if isinstance(op, Monoid): # A Monoid takes one type, but used element-wise it is its # BinaryOp, which takes the pair: ``v.ewise_mult(w, monoid.plus)`` - # works where ``v * w`` does. Indexing (``monoid.plus[X, Y]``) - # still raises. - return get_typed_op(op.binaryop, dtype, dtype2) + # works where ``v * w`` does. + if kind == "binary": + return get_typed_op(op.binaryop, dtype, dtype2) + # Indexed with the pair, it takes the type the pair promotes to. + return op[_monoid_pair_type(op.binaryop.name, dtype, dtype2)] return op._compile_udt(dtype, dtype2) # Generic case: try to unify the two dtypes try: diff --git a/graphblas/core/scalar.py b/graphblas/core/scalar.py index bb7339ce8..a7cf2189b 100644 --- a/graphblas/core/scalar.py +++ b/graphblas/core/scalar.py @@ -158,12 +158,12 @@ def __eq__(self, other): Without Numba, which ``binary.eq`` needs for UDTs, a UDT's values are compared in numpy, which broadcasts them the same way. """ - return _scalar_eq(self, other) + return _scalar_eq(self, other, "__eq__") __hash__ = None def __ne__(self, other): - return not _scalar_eq(self, other) + return not _scalar_eq(self, other, "__ne__") def __bool__(self): """Truthiness check. @@ -750,6 +750,7 @@ def ewise_mult(self, other, op=binary.times): def _ewise_mult(self, other, op=binary.times, is_infix=False): method_name = "ewise_mult" + given = other if is_infix: from .infix import ScalarEwiseMultExpr @@ -778,7 +779,7 @@ def _ewise_mult(self, other, op=binary.times, is_infix=False): ) op = get_typed_op(op, self.dtype, other.dtype, kind="binary") self._expect_op(op, ("BinaryOp", "Monoid"), within=method_name, argname="op") - return ScalarExpression( + expr = ScalarExpression( method_name, f"GrB_Vector_eWiseMult_{op.opclass}", [self._as_vector(), other._as_vector()], @@ -786,6 +787,9 @@ def _ewise_mult(self, other, op=binary.times, is_infix=False): is_cscalar=False, scalar_as_vector=True, ) + if output_type(given) is not Scalar: + expr._right_literal = given # a literal, as Scalar comparisons give it + return expr def ewise_union(self, other, op, left_default, right_default): """Perform element-wise computation on the union of sparse values, @@ -940,7 +944,7 @@ def apply(self, op, right=None, *, left=None): """ expr = self._as_vector().apply(op, right, left=left) - return ScalarExpression( + rv = ScalarExpression( expr.method_name, expr.cfunc_name, expr.args, @@ -950,6 +954,8 @@ def apply(self, op, right=None, *, left=None): is_cscalar=False, scalar_as_vector=True, ) + rv._right_literal = expr._right_literal + return rv def select(self, op, thunk=None): expr = self._as_vector().select(op, thunk) @@ -1137,8 +1143,11 @@ def dup(self, dtype=None, *, clear=False, is_cscalar=False, name=None, **opts): # End auto-generated code: Scalar -def _scalar_eq(scalar, other): - """Return ``scalar == other`` as a bool, by ``binary.eq`` (see ``Scalar.__eq__``).""" +def _scalar_eq(scalar, other, within): + """Return ``scalar == other`` as a bool, by ``binary.eq`` (see ``Scalar.__eq__``). + + ``within`` is the method an error names, ``__eq__`` or ``__ne__``. + """ if other is None: return scalar._is_empty if type(other) is not Scalar and output_type(other) is Scalar: @@ -1175,6 +1184,19 @@ def _scalar_eq(scalar, other): # float16) holds one value, which numpy compares as binary.eq does, with no # GraphBLAS call; and numpy compares float16, which Numba cannot. return _udt_eq_without_numba(scalar, other) + if type(other) is not Scalar: + # As ewise_mult converts it, but an error names the method the user called. + value, dtype = _literal_operand(scalar.dtype, other, binary.eq) + try: + other = Scalar.from_value(value, dtype, is_cscalar=False, name="s_eq_other") + except TypeError: + other = scalar._expect_type( + other, + Scalar, + within=within, + argname="other", + extra_message="Literal scalars also accepted.", + ) return bool(scalar.ewise_mult(other, binary.eq).new(name="s_eq").value) diff --git a/graphblas/core/udt.py b/graphblas/core/udt.py index 94e7b3136..27ebc7b90 100644 --- a/graphblas/core/udt.py +++ b/graphblas/core/udt.py @@ -80,11 +80,12 @@ def _iter_record_leaves(np_type, python_prefix="", c_prefix=""): ``("['outer']['inner_a']", "outer.inner_a", float64_dtype)``. Only record-in-record nesting is recognised. Array-typed fields are - yielded as a single leaf, which the Numba path handles: the generated - expression operates on the whole sub-array and the wrapper slice-assigns - it back. The JIT path never sees such a record, because a subarray dtype - has no entry in ``udt_codegen.NP_TO_C_TYPES`` and ``udt_codegen._udt_c_typedef`` - bails out, so the op keeps the cfunc for every call. + yielded as a single leaf, which the Numba path handles: a binary op + computes it element by element, a unary op applies its expression to the + whole sub-array, and the wrapper slice-assigns the result back. The JIT + path never sees such a record, because a subarray dtype has no entry in + ``udt_codegen.NP_TO_C_TYPES`` and ``udt_codegen._udt_c_typedef`` bails out, + so the op keeps the cfunc for every call. """ for name in np_type.names: field_dtype = np_type.fields[name][0] @@ -300,6 +301,20 @@ def _udt_op_types(op_name, dtype, dtype2): return lookup_dtype(result), [(in_x, in_y)] +def _monoid_pair_type(binaryop_name, dtype, dtype2): + """Return the type a built-in Monoid indexed with a UDT pair takes, compiling nothing. + + It is the type the pair promotes to, as ``monoid.plus["INT8", "INT16"]`` is + ``monoid.plus["INT16"]``: the type lifted arithmetic gives the pair, leaf by + leaf. ``binary.any`` keeps the second operand's type, so ``monoid.any`` + promotes as ``plus`` does, the same in either order. A pair that does not + pair raises the ``KeyError`` compiling the op would. + """ + op_name = binaryop_name if binaryop_name in BUILTIN_UDT_BINARY_OPS else "plus" + _check_udt_pair(op_name, dtype, dtype2, _get_udt_info(dtype), _get_udt_info(dtype2)) + return _udt_op_types(op_name, dtype, dtype2)[0] + + def _promoted_record(records, result_leaves): """Build a record laid out like ``records`` with ``result_leaves`` for its leaves. @@ -430,7 +445,7 @@ def _casts_safely_into(literal_type, dtype): return reason is None and all(np.can_cast(x, z, "safe") for x, z in pairs) -def _store_cast_op(result_dtype, output_dtype): +def _store_cast_op(result_dtype, output_dtype, *, is_value=False): """Return the UnaryOp that converts a result for an object of ``output_dtype``, or ``None``. GraphBLAS casts between built-in types on store (C's casts, with a float @@ -442,6 +457,8 @@ def _store_cast_op(result_dtype, output_dtype): UDTs whose layouts correspond (:func:`_udt_cast_error`). Any other pair raises ``DomainMismatch``. ``None`` means GraphBLAS stores the result as it is. This is the one place that decides what may be stored where. + ``is_value`` says the error is for an object stored or accumulated as it + is, not a computed result. """ if result_dtype is output_dtype or not (result_dtype._is_udt or output_dtype._is_udt): return None @@ -453,6 +470,11 @@ def _store_cast_op(result_dtype, output_dtype): return _udt_cast_op(result_dtype, output_dtype) elif _has_numba: reason = "a UDT casts only to another UDT" + if is_value: + raise DomainMismatch( + f"cannot store a value of type {result_dtype} in an object of type " + f"{output_dtype}: {reason}." + ) raise DomainMismatch( f"cannot store a result of type {result_dtype} in an object of type " f"{output_dtype}: {reason}. Use .new() for an object of the result's type, or " @@ -530,11 +552,14 @@ def _cast_for_store(value, dtype): another UDT type needs a converted copy: one element for a Scalar, which is made here, but the whole value for a Vector or Matrix, which raises. """ - if (cast_op := _store_cast_op(value.dtype, dtype)) is None: - return value from .base import BaseType from .scalar import Scalar + # An expression (``v[:] << w.apply(op)``) is a computed result, not a value. + is_value = isinstance(value, BaseType) + if (cast_op := _store_cast_op(value.dtype, dtype, is_value=is_value)) is None: + return value + if output_type(value) is not Scalar: raise DomainMismatch( f"cannot assign a value of type {value.dtype} into an object of type {dtype}: " diff --git a/graphblas/core/vector.py b/graphblas/core/vector.py index f4b72fca4..8e07f2bc2 100644 --- a/graphblas/core/vector.py +++ b/graphblas/core/vector.py @@ -1408,6 +1408,7 @@ def apply(self, op, right=None, *, left=None): """ method_name = "apply" + right_literal = None extra_message = ( "apply only accepts UnaryOp with no scalars or BinaryOp with `left` or `right` scalar" "or IndexUnaryOp with `right` thunk." @@ -1477,6 +1478,7 @@ def apply(self, op, right=None, *, left=None): expr_repr = "{1.name}.apply({op}, left={0._expr_name})" elif left is None: if type(right) is not Scalar: + right_literal = right right, dtype = _literal_operand(self.dtype, right, op) try: right = Scalar.from_value(right, dtype, is_cscalar=None, name="") @@ -1520,7 +1522,7 @@ def apply(self, op, right=None, *, left=None): expr_repr = "{0.name}.apply({op}, right={1._expr_name})" else: raise TypeError("Cannot provide both `left` and `right` to apply") - return VectorExpression( + expr = VectorExpression( method_name, cfunc_name, args, @@ -1528,6 +1530,8 @@ def apply(self, op, right=None, *, left=None): expr_repr=expr_repr, size=self._size, ) + expr._right_literal = right_literal + return expr def select(self, op, thunk=None): """Create a new Vector by applying ``op`` to each element of the Vector and keeping diff --git a/graphblas/select/__init__.py b/graphblas/select/__init__.py index b55766ff8..876b3ec6a 100644 --- a/graphblas/select/__init__.py +++ b/graphblas/select/__init__.py @@ -3,6 +3,9 @@ # identical to IndexUnaryOp under the hood, except they must return # boolean and they default to `select` when called, rather than # `apply`). +import math +import numbers + _delayed = {} _binary_to_select = {} @@ -42,21 +45,6 @@ def _resolve_expr(expr, callname, opname): ) tensor = expr.args[0] thunk = expr.args[1] - method = f"{opname}{expr.op.name}" - if method not in globals(): - # TODO: remove this once rowlt/rowge/collt/colge exist - # Convert thunk to Python int to avoid possible subtraction with uints - thunk = thunk.value - # Attempt to convert < into <= (rowlt is not part of official spec, but rowle is) - if expr.op.name == "lt": - method = f"{opname}le" - thunk -= 1 - # Attempt to convert >= into > (rowge is not part of official spec, but rowgt is) - elif expr.op.name == "ge": - method = f"{opname}gt" - thunk -= 1 - if method not in globals(): # pragma: no cover (sanity) - raise ValueError(f"Unknown or unregistered select method: {method}") if expr._is_scalar: # Handle ScalarExpressions that change their arguments to Vector if tensor._parent is not None: # e.g., suitesparse @@ -65,9 +53,55 @@ def _resolve_expr(expr, callname, opname): else: # e.g., suitesparse-vanilla tensor = tensor[0].new() thunk = thunk[0].new() + name = expr.op.name + if opname != "value": + if name in {"lt", "ge"} and f"{opname}{name}" in globals(): + # An op registered for it (a user's rowlt): the number as given + if expr._right_literal is not None: + thunk = expr._right_literal + else: + thunk, name = _index_bound(expr._right_literal, thunk, name) + method = f"{opname}{name}" + if method not in globals(): # pragma: no cover (sanity) + raise ValueError(f"Unknown or unregistered select method: {method}") return globals()[method](tensor, thunk) +def _index_bound(literal, thunk, name): + """Return the thunk and comparison that select an index ``name`` (lt, le, gt, ge) a bound. + + The bound is the number the caller gave (``literal``), else the thunk Scalar's value. + The comparison typed its thunk beside the values, which is wrong for an index: 300 + beside INT8 values is infinity, and 2**24 + 1 beside FP32 rounds. Only rowle and + rowgt exist (and colle, colgt, indexle, indexgt), and indices are whole numbers, so + ``i < t`` is ``i <= ceil(t) - 1``, ``i >= t`` is ``i > ceil(t) - 1``, and ``i <= t`` + and ``i > t`` take ``floor(t)``, where GraphBLAS would truncate a float toward zero + (``i > -0.5`` dropped index 0). A whole number beyond INT64, which holds every index, + is clamped to it. NaN compares False with every index, so it keeps none. An empty + Scalar goes to the op as it is, which raises EmptyObject, and so does a thunk that is + no Scalar (from ``A.apply(binary.le, left=3)``), which raises the op's TypeError. + """ + shift = name in {"lt", "ge"} + if shift: + name = "le" if name == "lt" else "gt" + # The caller's number, if it is one; a Scalar expression (``w[0]``), a 0-d array or + # ``np.True_`` made the thunk keep its own type, so the thunk's value is the bound. + if isinstance(literal, numbers.Real): + bound = literal + elif getattr(thunk, "_is_scalar", False): + bound = thunk.value + else: + return thunk, name + if not isinstance(bound, numbers.Real): + return thunk, name + if math.isnan(bound): + return (-1 if name == "le" else 2**63 - 1), name # no index is <= -1 or > INT64_MAX + if math.isfinite(bound): + bound = math.ceil(bound) - 1 if shift else math.floor(bound) + bound = min(max(bound, -(2**63)), 2**63 - 1) + return bound, name + + def _match_expr(parent, expr): """Match expressions to rewrite ``A.select(A < 5)`` into select expression. diff --git a/graphblas/tests/test_dtype.py b/graphblas/tests/test_dtype.py index 8ca033625..e2df81017 100644 --- a/graphblas/tests/test_dtype.py +++ b/graphblas/tests/test_dtype.py @@ -378,10 +378,12 @@ def test_register_anonymous_zero_size(): def test_register_anonymous_array_numba_cannot_type(): - # Numba cannot type an array of these, alone or in a record. Its own error was - # "AttributeError: 'Boolean' object has no attribute 'bitwidth'" and the like. + # Numba cannot type an array of these, alone, in a record or in an array of records. + # Its own error was "AttributeError: 'Boolean' object has no attribute 'bitwidth'" + # and the like. for np_type, element, instead in [ (np.dtype((np.bool_, (2,))), "bool", "uint8"), + (np.dtype(([("_nbt_e", np.bool_, (2,))], (3,))), "bool", "uint8"), (np.dtype(("M8[s]", (2, 3))), "datetime64", "int64"), (np.dtype([("_nbt_a", np.int8), ("_nbt_b", "m8[ns]", (2,))]), "timedelta64", "int64"), (np.dtype(("U3", (2,))), "str", "uint32"), diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index d1405c49f..306f9837b 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -1273,6 +1273,57 @@ def test_select(A): assert pickle.loads(pickle.dumps(select.tril[bool])) is select.tril[bool] +def test_select_row_and_column_compare_indices_with_the_bound_given(): + # The comparison types its literal beside the values, where 300 beside INT8 or UINT8 + # is infinity and 2**24 + 1 beside FP32 rounds; the helpers used that for indices + # (row(B < 300) kept all 400 rows, row(B > 299) none; main was right). + n = 400 + for dtype in ["INT8", "UINT8"]: + B = Matrix.from_coo(np.arange(n), np.zeros(n, int), 1, dtype=dtype, nrows=n, ncols=1) + assert select.row(B < 300).new().nvals == 300 + assert select.row(B > 299).new().nvals == 100 + assert select.row(B <= 299).new().nvals == 300 + assert select.row(B >= 300).new().nvals == 100 + assert select.column(B.T <= 299).new().nvals == 300 + assert select.column(B.T > 299).new().nvals == 100 + assert select.row(B < 2**70).new().nvals == select.row(B > -(2**70)).new().nvals == n + rows = [2**24 - 1, 2**24, 2**24 + 1] + F = Matrix.from_coo(rows, [0, 0, 0], [1.0, 2.0, 3.0], dtype="FP32", nrows=2**24 + 3, ncols=1) + assert select.row(F < 2**24 + 1).new().to_coo()[0].tolist() == rows[:2] + # A fractional bound: i < 2.5 keeps 2, i > -0.5 keeps 0 (a float thunk was truncated) + A = Matrix.from_coo(np.arange(5), np.zeros(5, int), 1.0, nrows=5, ncols=1) + for expr, expected in [ + (A < A.nrows / 2, [0, 1, 2]), + (A <= 2.5, [0, 1, 2]), + (A >= 2.5, [3, 4]), + (A > 2.5, [3, 4]), + (A > -0.5, [0, 1, 2, 3, 4]), + (A < -0.5, []), + (A < float("inf"), [0, 1, 2, 3, 4]), + (A < Scalar.from_value(2.5), [0, 1, 2]), + # A bound that is not a Python or numpy number: the thunk keeps its own type + (A < np.array(3), [0, 1, 2]), + (A >= np.array(3), [3, 4]), + (A < np.True_, [0]), + ]: + assert select.row(expr).new().to_coo()[0].tolist() == expected, expr + assert select.column(A.T < 2.5).new().to_coo()[1].tolist() == [0, 1, 2] + w = Vector.from_coo([0], [3]) + with gb.config.set(autocompute=True): + assert select.row(A < w[0]).new().to_coo()[0].tolist() == [0, 1, 2] + assert select.row(A >= w[0]).new().to_coo()[0].tolist() == [3, 4] + # A comparison with the scalar on the left gives the op's TypeError, as on main; it + # was AttributeError ('Matrix' object has no attribute 'value') + with pytest.raises(TypeError, match="Bad type for keyword argument `thunk=`"): + select.row(A.apply(binary.le, left=3)) + # NaN compares False with every index, so it keeps none; NaN cast to the INT64 thunk + # was 0, so < and <= kept row 0, and > and >= every other row (main too) + nan = float("nan") + for expr in [A < nan, A <= nan, A > nan, A >= nan, A < Scalar.from_value(nan)]: + assert select.row(expr).new().nvals == 0, expr + assert select.column(A.T >= np.float32(nan)).new().nvals == 0 + + @autocompute def test_select_bools_and_masks(A): A3 = Matrix.from_coo([0, 3, 3, 6], [3, 0, 2, 4], [3, 3, 3, 3], nrows=7, ncols=7) diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index d51993c2e..3c72e36c2 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -4576,6 +4576,7 @@ def test_udt_literal_converts_only_when_exact(): np.dtype([("ex_v", np.int8, (2,)), ("ex_w", np.int8)]), "_ExactNested" ) text = dtypes.register_anonymous(np.dtype([("ex_s", "S4"), ("ex_n", np.int8)]), "_ExactText") + char = dtypes.register_anonymous(np.dtype([("ex_c", "S1"), ("ex_k", np.int8)]), "_ExactChar") bare = dtypes.register_anonymous(np.dtype("S3"), "_ExactBareText") seconds = dtypes.register_anonymous(np.dtype("m8[s]"), "_ExactBareSeconds") dates = dtypes.register_anonymous( @@ -4593,6 +4594,8 @@ def test_udt_literal_converts_only_when_exact(): n[0] = ([1, 2], 3) t = Vector(text, size=1) t[0] = (b"ab", 1) + c = Vector(char, size=1) + c[0] = (b"a", 1) b = Vector(bare, size=1) b[0] = b"0.5" d = Vector(seconds, size=1) @@ -4656,6 +4659,8 @@ def test_udt_literal_converts_only_when_exact(): (lambda: b.apply(same, right=np.True_), "_ExactBareText: it is typed as BOOL,"), (lambda: b.apply(same, right=b"0.5000"), "_ExactBareText"), # was cut to b"0.5" (lambda: t.apply(second, right=cut_text), "_ExactText"), + # NaT as b"N" + (lambda: c.apply(second, right=(np.datetime64("NaT", "s"), 1)), "_ExactChar"), (lambda: d.apply(same, right=np.timedelta64(5, "ms")), "_ExactBareSeconds"), ]: with pytest.raises(ValueError, match=re.escape(f"does not fit {typed_as}")): @@ -5004,11 +5009,32 @@ def test_udt_record_date_literals_compare_as_numpy(): ((datetime.date(2020, 1, 1), datetime.timedelta(seconds=5)), True), ((midnight, secs), True), (("2020-01-01", secs), True), + # Text in a duration field takes the field's type, as before. + ((day, "5"), True), ]: assert (s == literal) is equal if supports_udfs: assert (v == literal).new()[0].new().value == equal assert (v != literal).new()[0].new().value != equal + # Text numpy reads as NaT is NaT in the field's unit (numpy 2.5 warns for NaT with no + # unit), and NaT equals nothing, as in numpy. + bare = dtypes.register_anonymous(np.dtype("M8[s]"), "_DateLitBare") + b = gb.Scalar.from_value(np.datetime64(0, "s"), dtype=bare) + for nat in ["NaT", "nat", "", b"NaT"]: + assert (s == (nat, secs)) is False + assert (b == nat) is False + if supports_udfs: + assert not (v == (nat, secs)).new()[0].new().value + # So does text numpy cannot read as a date or duration, and converting it says why. + for literal, reason in [ + (("not a date", secs), "Error parsing datetime string"), + ((day, "abc"), "Could not convert object to NumPy timedelta"), + ]: + with pytest.raises(ValueError, match=reason): + s == literal # noqa: B015 + if supports_udfs: + with pytest.raises(ValueError, match=reason): + (v == literal).new() if supports_udfs: # A user's op refuses what its field would cut short, as for a numpy date. second = BinaryOp.register_anonymous(lambda x, y: y, "_date_lit_second", is_udt=True) @@ -5839,8 +5865,11 @@ def test_udt_store_casts_reach_every_store_path(): u = Vector(f8s, size=3) u[[0, 2]] << unary.ainv(gb.Scalar.from_value([1, 2, 3], dtype=i8s)) assert u.to_coo()[1].tolist() == [[-1, -2, -3]] * 2 - with pytest.raises(DomainMismatch, match="a UDT casts only to another UDT"): + with pytest.raises(DomainMismatch, match="result of type .*casts only to another UDT. Use"): gb.Scalar(FP64) << v.reduce(agg.sum) + # Assigned to part of an object, a computed result is still a result. + with pytest.raises(DomainMismatch, match="result of type FP64 .*another UDT. Use"): + v[:] << Vector.from_coo([0], [0.5], size=3).apply(unary.ainv) w = Vector(f8s, size=3) w[0] = [0.5, 0.5, 0.5] w[1] = [1.5, 1.5, 1.5] @@ -5902,9 +5931,14 @@ def added_to_int8(value): s.clear() s(accum=binary.plus) << 0.5 assert s.value.tolist() == [0, 0, 0] - # A built-in Scalar casts only into a built-in type, as for a store. - with pytest.raises(DomainMismatch, match="a UDT casts only to another UDT"): + # A built-in Scalar casts only into a built-in type, as for a store. The error + # calls it a value, not a result (it said "a result of type FP64"). + with pytest.raises(DomainMismatch, match="value of type FP64 .*casts only to another UDT.$"): added(gb.Scalar.from_value(0.5)) + # A Vector is not a value to accumulate, into a UDT Scalar or a built-in one. + for accumulate in [added, added_to_int8]: + with pytest.raises(TypeError, match="Bad type for keyword argument `other=`"): + accumulate(halves) @pytest.mark.skipif("not supports_udfs") @@ -6333,7 +6367,9 @@ def test_udt_pair_is_never_unified(): which ``unify`` registered as a new type. An op that does not lift to UDTs then named that third type in its KeyError, and ``ewise_add`` with a monoid passed GraphBLAS two operands it rejected with GrB_DOMAIN_MISMATCH. A - Monoid takes one type, so on a mixed pair it runs as its BinaryOp. + Monoid takes one type, so on a mixed pair it runs as its BinaryOp, and + indexed with the pair it takes the type the pair promotes to, as + ``monoid.plus["INT8", "INT16"]`` is ``monoid.plus["INT16"]``. """ X = dtypes.register_anonymous(np.dtype([("uni_a", np.float32), ("uni_b", np.int64)]), "_UniX") Y = dtypes.register_anonymous(np.dtype([("uni_a", np.int64), ("uni_b", np.float32)]), "_UniY") @@ -6352,8 +6388,22 @@ def test_udt_pair_is_never_unified(): for expr in [lambda: v.ewise_add(w), lambda: monoid.plus(v | w)]: with pytest.raises(DomainMismatch, match="ewise_add cannot use binary.plus"): expr() - with pytest.raises(TypeError, match="Monoid inputs must be the same dtype"): - monoid.plus[X, Y] + # Indexed with the pair: TypeError (a Monoid takes one type) before; main and + # built-in types promote. + XY = binary.plus[X, Y].return_type + assert monoid.plus[X, Y] is monoid.plus[Y, X] is monoid.plus[XY] + assert monoid.max[X, Y] is monoid.max[XY] + R8 = dtypes.register_anonymous(np.dtype([("uni_c", np.int8)]), "_UniR8") + R16 = dtypes.register_anonymous(np.dtype([("uni_c", np.int16)]), "_UniR16") + assert monoid.plus[R8, R16] is monoid.plus[R16] + assert monoid.plus[R8, "INT16"] is monoid.plus[R16] + # binary.any gives the second operand's type, but monoid.any promotes the pair in + # either order, as main did + for pair in [(R8, R16), (R16, R8), (R8, "INT16"), ("INT16", R8)]: + assert monoid.any[pair] is monoid.any[R16], pair + assert monoid.any[X, Y] is monoid.any[Y, X] is monoid.any[XY] + with pytest.raises(KeyError, match="plus does not work with"): + monoid.plus[X, R8] @pytest.mark.skipif("not supports_udfs") diff --git a/graphblas/tests/test_operator_types.py b/graphblas/tests/test_operator_types.py index 027f02fcc..8c250bc25 100644 --- a/graphblas/tests/test_operator_types.py +++ b/graphblas/tests/test_operator_types.py @@ -204,6 +204,7 @@ "binary_pickle", "monoid_pickle", "semiring_pickle", + "_test_weak_lt", # numpy-graphblas commutation (can we clean this up?) "band_land", "band_lor", diff --git a/graphblas/tests/test_scalar.py b/graphblas/tests/test_scalar.py index 686e4861a..3ade2fb25 100644 --- a/graphblas/tests/test_scalar.py +++ b/graphblas/tests/test_scalar.py @@ -293,7 +293,10 @@ def test_udt_nat_duration_in_another_unit_stays_nat(): np.dtype([("nat_o", [("nat_d", "m8[s]")]), ("nat_n", np.int8)]), "_NatNested" ) # Numba cannot type an array of durations, so registering one needs it absent. - pair = None if gb.core._has_numba else dtypes.register_anonymous(np.dtype(("m8[s]", 2))) + pair = pairs = None + if not gb.core._has_numba: + pair = dtypes.register_anonymous(np.dtype(("m8[s]", 2))) + pairs = dtypes.register_anonymous(np.dtype([("nat_p", "m8[s]", 2), ("nat_n", np.int8)])) for unit in ["ns", "ms", "m", "D"]: nat = np.timedelta64("NaT", unit) assert np.isnat(Scalar.from_value(nat, seconds).value) @@ -321,6 +324,9 @@ def test_udt_nat_duration_in_another_unit_stays_nat(): got = Scalar.from_value([nat, np.timedelta64(5000, "ms")], pair).value assert np.isnat(got[0]) assert got[1] == np.timedelta64(5, "s") + got = Scalar.from_value(([nat, np.timedelta64(5000, "ms")], 1), pairs).value + assert np.isnat(got["nat_p"][0]) + assert got["nat_p"][1] == np.timedelta64(5, "s") r = Vector(days, size=1) r[0] = (np.datetime64("NaT", unit), 1) assert np.isnat(r[0].new().value["nat_t"]) @@ -462,6 +468,17 @@ def test_eq_types_a_literal_as_binary_eq_does(): assert empty == Scalar(dtypes.FP64) assert empty == None # noqa: E711 assert empty != 0 + # A value that is not a literal names the operator used, not Scalar.ewise_mult. + s = Scalar.from_value(1, dtype=dtypes.INT8) + with pytest.raises(TypeError, match=r"argument `other` in Scalar\.__eq__\(\.\.\.\)"): + s == [1] # noqa: B015 + with pytest.raises(TypeError, match=r"argument `other` in Scalar\.__ne__\(\.\.\.\)"): + s != object() # noqa: B015 + # A dict is not an element of an array UDT (AttributeError from reading its fields) + a = Scalar.from_value([1, 2, 3], dtypes.register_anonymous(np.dtype((np.int8, 3)))) + match = r"Scalar\.__eq__" if gb.core._supports_udfs else None + with pytest.raises(TypeError, match=match): + a == {} # noqa: B015 @autocompute @@ -717,6 +734,19 @@ def test_ewise_union(s): s.ewise_union(t, binary.plus, 10, v) +def test_select_row_compares_its_index_with_the_bound_given(s): + # A Scalar's only index is 0. The bound is the number given, made a whole number + # (s > -0.5 dropped it, and s < 1 raised AttributeError), not the comparison's thunk + # typed beside the value (300 beside INT8 is infinity). + assert select.row(s > -0.5).new() == s + assert select.row(s < 0.5).new() == s + assert select.row(s < 1).new() == s + assert select.row(s >= 0.5).new().is_empty + s8 = Scalar.from_value(1, dtype=dtypes.INT8) + assert select.row(s8 < 300).new() == s8 + assert select.row(s8 > 299).new().is_empty + + def test_ewise_mult_add(s): assert s.ewise_add(s).new() == 10 assert s.ewise_mult(s).new() == 25 diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 7d5cab524..3fa48e661 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -751,6 +751,11 @@ def test_select(v): w8 = select.index(v >= 4).new() expected = Vector.from_coo([4, 6], [2, 0], size=7) assert w8.isequal(expected) + # The index is compared with the bound given, not with 300 typed beside INT8 (inf) + w = Vector.from_coo([0, 200, 399], 1, dtype=dtypes.INT8, size=400) + assert select.index(w < 300).new().to_coo()[0].tolist() == [0, 200] + assert select.index(w >= 300).new().to_coo()[0].tolist() == [399] + assert select.index(w > 2.5).new().to_coo()[0].tolist() == [200, 399] @autocompute @@ -1608,6 +1613,59 @@ def test_python_number_is_weak_beside_builtin_dtypes(): plus = binary.register_anonymous(lambda x, y: x + y) with pytest.raises(OverflowError, match="out of bounds"): i8.apply(plus, right=300) + # Any op whose result is another type, as truediv's and a comparison's are, takes + # an int out of range exactly: a parameterized op, a user's, and the instances a + # parameterized op makes (these raised OverflowError; main and numpy give False). + gt = binary.register_anonymous(lambda x, y: x > y) + for op, values in [ + (binary.isclose, [False, False]), + (binary.isclose(0.1), [False, False]), + (gt, [False, False]), + ]: + assert i8.apply(op, right=300).new().to_coo()[1].tolist() == values + assert i8.apply(op, left=-300).new().to_coo()[1].tolist() == values + result = i8.ewise_union(w8, binary.isclose, 0, 300).new() + assert result.to_coo()[1].tolist() == [False, False] + # A user's op named like a comparison is not one: the int is not made infinity + # (it gave [-inf, -inf] and [True, True]; main gives these, as numpy would) + + def ge(x, y): + return x * 1.0 - y + + def eq(x, y): + return y - x > 250 + + result = i8.apply(binary.register_anonymous(ge), right=300).new() + assert result.to_coo()[1].tolist() == [-200.0, -400.0] + result = i8.apply(binary.register_anonymous(eq), right=300).new() + assert result.to_coo()[1].tolist() == [False, True] + # Where the op and the int would compute in a float type, which rounds an int + # beyond 64 bits, a user's op raises (it gave [False, True]: 2**63 - 1 < 2**63 is + # False in FP64). A built-in op that computes in float anyway does not. + i64 = Vector.from_coo([0, 1], [2**63 - 1, 0], dtype=dtypes.INT64) + u64 = Vector.from_coo([0, 1], [2**64 - 1, 0], dtype=dtypes.UINT64) + lt = binary.register_anonymous(lambda x, y: x < y) + if not hasattr(binary, "_test_weak_lt"): + binary.register_new("_test_weak_lt", lambda x, y: x < y) + for vec, value in [(i64, 2**63), (i64, -(2**63) - 1), (u64, 2**64)]: + for op in [lt, binary._test_weak_lt]: + with pytest.raises(OverflowError, match="out of bounds"): + vec.apply(op, right=value) + assert vec.apply(binary.lt, right=value).new().to_coo()[1].tolist() == [value > 0] * 2 + result = i64.apply(binary.isclose, right=2**63).new() + assert result.to_coo()[1].tolist() == [True, False] # as numpy's isclose + # Built-in ops made outside register_new (truediv) or on first use (binom) count + # as built-in too: truediv computes beside a 64-bit int, as numpy divides + # (these raised OverflowError). + assert (i64 / 2**63).new().to_coo()[1].tolist() == [1.0, 0.0] + assert (u64 / -1).new().to_coo()[1].tolist() == [-(2.0**64), 0.0] + assert (i8 / 2**70).new().to_coo()[1].tolist() == [100 / 2**70, -100 / 2**70] + from graphblas.core.literals import _is_builtin_op + + assert all( + _is_builtin_op(op) + for op in [binary.truediv, binary.rtruediv, binary.cdiv, binary.binom] + ) # Typed scalars promote as numpy 2 promotes them, on numpy 1 too. assert dtypes.unify(dtypes.INT8, dtypes.INT64, is_right_scalar=True) == dtypes.INT64