From cc49a8e1df66ce45c3d5e0936313d0763198bb7f Mon Sep 17 00:00:00 2001 From: Jim Kitchen Date: Fri, 13 May 2022 17:43:07 -0500 Subject: [PATCH 01/11] UDF for IndexUnaryOp and SelectOp --- graphblas/__init__.py | 3 + graphblas/binary/__init__.py | 3 - graphblas/binary/numpy.py | 6 +- graphblas/indexunary/__init__.py | 3 - graphblas/monoid/__init__.py | 3 - graphblas/monoid/numpy.py | 6 +- graphblas/op/__init__.py | 3 - graphblas/operator.py | 424 ++++++++++++++++++++++++++++--- graphblas/select/__init__.py | 3 - graphblas/semiring/__init__.py | 3 - graphblas/semiring/numpy.py | 6 +- graphblas/tests/test_op.py | 19 +- graphblas/unary/__init__.py | 3 - graphblas/unary/numpy.py | 6 +- 14 files changed, 430 insertions(+), 61 deletions(-) diff --git a/graphblas/__init__.py b/graphblas/__init__.py index 5619be433..4a6db6de3 100644 --- a/graphblas/__init__.py +++ b/graphblas/__init__.py @@ -156,6 +156,9 @@ def _init(backend_arg, blocking, automatic=False): _init_params = passed_params +# Ideally this is in operator.py, but lives here to avoid circular references +_STANDARD_OPERATOR_NAMES = set() + _NEEDS_OPERATOR = { "graphblas._agg", "graphblas.agg", diff --git a/graphblas/binary/__init__.py b/graphblas/binary/__init__.py index f18b04bc3..899d6c7a5 100644 --- a/graphblas/binary/__init__.py +++ b/graphblas/binary/__init__.py @@ -8,11 +8,8 @@ "rfloordiv": "floordiv", "rpow": "pow", } -from graphblas import operator # noqa isort:skip from . import numpy # noqa isort:skip -del operator - def __dir__(): return globals().keys() | _delayed.keys() diff --git a/graphblas/binary/numpy.py b/graphblas/binary/numpy.py index 1af2a5574..5740611ea 100644 --- a/graphblas/binary/numpy.py +++ b/graphblas/binary/numpy.py @@ -7,9 +7,9 @@ """ import numpy as _np +from .. import _STANDARD_OPERATOR_NAMES from .. import binary as _binary from .. import config as _config -from .. import operator as _operator _delayed = {} _binary_names = { @@ -56,7 +56,7 @@ "nextafter", "ldexp", } -_operator._STANDARD_OPERATOR_NAMES.update(f"binary.numpy.{name}" for name in _binary_names) +_STANDARD_OPERATOR_NAMES.update(f"binary.numpy.{name}" for name in _binary_names) __all__ = list(_binary_names) _numpy_to_graphblas = { # Monoids @@ -140,6 +140,8 @@ def __getattr__(name): if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_binary, _numpy_to_graphblas[name]) else: + from .. import operator as _operator + numpy_func = getattr(_np, name) _operator.BinaryOp.register_new(f"numpy.{name}", lambda x, y: numpy_func(x, y)) rv = globals()[name] diff --git a/graphblas/indexunary/__init__.py b/graphblas/indexunary/__init__.py index f051d518c..fd8611158 100644 --- a/graphblas/indexunary/__init__.py +++ b/graphblas/indexunary/__init__.py @@ -1,9 +1,6 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of IndexUnaryOp instances _delayed = {} -from graphblas import operator # noqa isort:skip - -del operator def __dir__(): diff --git a/graphblas/monoid/__init__.py b/graphblas/monoid/__init__.py index 622685b24..9181c7ac5 100644 --- a/graphblas/monoid/__init__.py +++ b/graphblas/monoid/__init__.py @@ -1,11 +1,8 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of Monoid instances _delayed = {} -from graphblas import operator # noqa isort:skip from . import numpy # noqa isort:skip -del operator - def __dir__(): return globals().keys() | _delayed.keys() diff --git a/graphblas/monoid/numpy.py b/graphblas/monoid/numpy.py index b9deea329..3f2513580 100644 --- a/graphblas/monoid/numpy.py +++ b/graphblas/monoid/numpy.py @@ -7,10 +7,10 @@ """ import numpy as _np +from .. import _STANDARD_OPERATOR_NAMES from .. import binary as _binary from .. import config as _config from .. import monoid as _monoid -from .. import operator as _operator from ..dtypes import _supports_complex _delayed = {} @@ -98,7 +98,7 @@ ) _monoid_identities["minimum"].update(dict.fromkeys(_complex_dtypes, complex(_np.inf, _np.inf))) -_operator._STANDARD_OPERATOR_NAMES.update(f"monoid.numpy.{name}" for name in _monoid_identities) +_STANDARD_OPERATOR_NAMES.update(f"monoid.numpy.{name}" for name in _monoid_identities) __all__ = list(_monoid_identities) _numpy_to_graphblas = { "add": "plus", @@ -136,6 +136,8 @@ def __getattr__(name): if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_monoid, _numpy_to_graphblas[name]) else: + from .. import operator as _operator + func = getattr(_binary.numpy, name) _operator.Monoid.register_new(f"numpy.{name}", func, _monoid_identities[name]) return globals()[name] diff --git a/graphblas/op/__init__.py b/graphblas/op/__init__.py index 56a071039..b214e4b58 100644 --- a/graphblas/op/__init__.py +++ b/graphblas/op/__init__.py @@ -1,11 +1,8 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of all UnaryOp, BinaryOp, and Semiring instances _delayed = {} -from graphblas import operator # noqa isort:skip from . import numpy # noqa isort:skip -del operator - def __dir__(): return globals().keys() | _delayed.keys() diff --git a/graphblas/operator.py b/graphblas/operator.py index a4665d23c..782b95297 100644 --- a/graphblas/operator.py +++ b/graphblas/operator.py @@ -9,7 +9,7 @@ import numba import numpy as np -from . import config, ffi, lib +from . import _STANDARD_OPERATOR_NAMES, config, ffi, lib from .dtypes import ( BOOL, FP32, @@ -34,8 +34,6 @@ if _supports_complex: from .dtypes import FC32, FC64 -_STANDARD_OPERATOR_NAMES = set() - from . import binary, indexunary, monoid, op, select, semiring, unary # noqa isort:skip ffi_new = ffi.new @@ -381,6 +379,42 @@ def _numba_func(self): __call__ = TypedBuiltinUnaryOp.__call__ +class TypedUserIndexUnaryOp(TypedOpBase): + __slots__ = () + opclass = "IndexUnaryOp" + + def __init__(self, parent, name, type_, return_type, gb_obj): + super().__init__(parent, name, type_, return_type, gb_obj, f"{name}_{type_}") + + @property + def orig_func(self): + return self.parent.orig_func + + @property + def _numba_func(self): + return self.parent._numba_func + + __call__ = TypedBuiltinIndexUnaryOp.__call__ + + +class TypedUserSelectOp(TypedOpBase): + __slots__ = () + opclass = "SelectOp" + + def __init__(self, parent, name, type_, return_type, gb_obj): + super().__init__(parent, name, type_, return_type, gb_obj, f"{name}_{type_}") + + @property + def orig_func(self): + return self.parent.orig_func + + @property + def _numba_func(self): + return self.parent._numba_func + + __call__ = TypedBuiltinSelectOp.__call__ + + class TypedUserBinaryOp(TypedOpBase): __slots__ = "_monoid" opclass = "BinaryOp" @@ -489,6 +523,70 @@ def _deserialize(name, func, anonymous): return UnaryOp.register_new(name, func, parameterized=True) +class ParameterizedIndexUnaryOp(ParameterizedUdf): + __slots__ = "func", "__signature__", "_is_udt" + + def __init__(self, name, func, *, anonymous=False, is_udt=False): + self.func = func + self.__signature__ = inspect.signature(func) + self._is_udt = is_udt + if name is None: + name = getattr(func, "__name__", name) + super().__init__(name, anonymous) + + def _call(self, *args, **kwargs): + indexunary = self.func(*args, **kwargs) + indexunary._parameterized_info = (self, args, kwargs) + return IndexUnaryOp.register_anonymous(indexunary, self.name, is_udt=self._is_udt) + + def __reduce__(self): + name = f"indexunary.{self.name}" + if not self._anonymous and name in _STANDARD_OPERATOR_NAMES: + return name + return (self._deserialize, (self.name, self.func, self._anonymous)) + + @staticmethod + def _deserialize(name, func, anonymous): + if anonymous: + return IndexUnaryOp.register_anonymous(func, name, parameterized=True) + rv = IndexUnaryOp._find(name) + if rv is not None: + return rv + return IndexUnaryOp.register_new(name, func, parameterized=True) + + +class ParameterizedSelectOp(ParameterizedUdf): + __slots__ = "func", "__signature__", "_is_udt" + + def __init__(self, name, func, *, anonymous=False, is_udt=False): + self.func = func + self.__signature__ = inspect.signature(func) + self._is_udt = is_udt + if name is None: + name = getattr(func, "__name__", name) + super().__init__(name, anonymous) + + def _call(self, *args, **kwargs): + sel = self.func(*args, **kwargs) + sel._parameterized_info = (self, args, kwargs) + return SelectOp.register_anonymous(sel, self.name, is_udt=self._is_udt) + + def __reduce__(self): + name = f"select.{self.name}" + if not self._anonymous and name in _STANDARD_OPERATOR_NAMES: + return name + return (self._deserialize, (self.name, self.func, self._anonymous)) + + @staticmethod + def _deserialize(name, func, anonymous): + if anonymous: + return SelectOp.register_anonymous(func, name, parameterized=True) + rv = SelectOp._find(name) + if rv is not None: + return rv + return SelectOp.register_new(name, func, parameterized=True) + + class ParameterizedBinaryOp(ParameterizedUdf): __slots__ = "func", "__signature__", "_monoid", "_cached_call", "_commutes_to", "_is_udt" @@ -1031,6 +1129,8 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal @classmethod def _initialize(cls): + if cls._initialized: + return super()._initialize() # Update type information with sane coercion position_dtypes = [ @@ -1121,12 +1221,12 @@ def __reduce__(self): class IndexUnaryOp(OpBase): - __slots__ = "is_positional" + __slots__ = "orig_func", "is_positional", "_is_udt", "_numba_func" _module = indexunary _modname = "indexunary" - _is_udt = False _custom_dtype = None _typed_class = TypedBuiltinIndexUnaryOp + _typed_user_class = TypedUserIndexUnaryOp _parse_config = { "trim_from_front": 4, "num_underscores": 1, @@ -1147,9 +1247,170 @@ class IndexUnaryOp(OpBase): "rowindex", "colindex"} # fmt: on + @classmethod + def _build(cls, name, func, *, is_udt=False, anonymous=False): + if not isinstance(func, FunctionType): + raise TypeError(f"UDF argument must be a function, not {type(func)}") + if name is None: + name = getattr(func, "__name__", "") + success = False + indexunary_udf = numba.njit(func) + new_type_obj = cls( + name, func, anonymous=anonymous, is_udt=is_udt, numba_func=indexunary_udf + ) + return_types = {} + nt = numba.types + if not is_udt: + for type_ in _sample_values: + sig = (type_.numba_type, UINT64.numba_type, UINT64.numba_type, type_.numba_type) + try: + indexunary_udf.compile(sig) + except numba.TypingError: + continue + ret_type = lookup_dtype(indexunary_udf.overloads[sig].signature.return_type) + if ret_type != type_ and ( + ("INT" in ret_type.name and "INT" in type_.name) + or ("FP" in ret_type.name and "FP" in type_.name) + or ("FC" in ret_type.name and "FC" in type_.name) + or (type_ == UINT64 and ret_type == FP64 and return_types.get(INT64) == INT64) + ): + # Downcast `ret_type` to `type_`. + # This is what users want most of the time, but we can't make a perfect rule. + # There should be a way for users to be explicit. + ret_type = type_ + elif type_ == BOOL and ret_type == INT64 and return_types.get(INT8) == INT8: + ret_type = INT8 + + # Numba is unable to handle BOOL correctly right now, but we have a workaround + # See: https://github.com/numba/numba/issues/5395 + # We're relying on coercion behaving correctly here + input_type = INT8 if type_ == BOOL else type_ + return_type = INT8 if ret_type == BOOL else ret_type + + # Build wrapper because GraphBLAS wants pointers and void return + wrapper_sig = nt.void( + nt.CPointer(return_type.numba_type), + nt.CPointer(input_type.numba_type), + UINT64.numba_type, + UINT64.numba_type, + nt.CPointer(input_type.numba_type), + ) + + if type_ == BOOL: + if ret_type == BOOL: + + def indexunary_wrapper(z, x, row, col, y): + z[0] = bool( + indexunary_udf(bool(x[0]), row, col, bool(y[0])) + ) # pragma: no cover + + else: + + def indexunary_wrapper(z, x, row, col, y): + z[0] = indexunary_udf( + bool(x[0]), row, col, bool(y[0]) + ) # pragma: no cover + + elif ret_type == BOOL: + + def indexunary_wrapper(z, x, row, col, y): + z[0] = bool(indexunary_udf(x[0], row, col, y[0])) # pragma: no cover + + else: + + def indexunary_wrapper(z, x, row, col, y): + z[0] = indexunary_udf(x[0], row, col, y[0]) # pragma: no cover + + indexunary_wrapper = numba.cfunc(wrapper_sig, nopython=True)(indexunary_wrapper) + new_indexunary = ffi_new("GrB_IndexUnaryOp*") + check_status_carg( + lib.GrB_IndexUnaryOp_new( + new_indexunary, + indexunary_wrapper.cffi, + ret_type.gb_obj, + type_.gb_obj, + type_.gb_obj, + ), + "IndexUnaryOp", + new_indexunary, + ) + op = cls._typed_user_class(new_type_obj, name, type_, ret_type, new_indexunary[0]) + new_type_obj._add(op) + success = True + return_types[type_] = ret_type + if success or is_udt: + return new_type_obj + else: + raise UdfParseError("Unable to parse function using Numba") + + def _compile_udt(self, dtype, dtype2): + if dtype2 is None: + dtype2 = dtype + dtypes = (dtype, dtype2) + if dtypes in self._udt_types: + return self._udt_ops[dtypes] + + numba_func = self._numba_func + sig = (dtype.numba_type, dtype2.numba_type) + numba_func.compile(sig) # Should we catch and give additional error message? + ret_type = lookup_dtype(numba_func.overloads[sig].signature.return_type) + indexunary_wrapper, wrapper_sig = _get_udt_wrapper(numba_func, ret_type, dtype, dtype2) + + indexunary_wrapper = numba.cfunc(wrapper_sig, nopython=True)(indexunary_wrapper) + new_indexunary = ffi_new("GrB_IndexUnaryOp*") + check_status_carg( + lib.GrB_IndexUnaryOp_new( + new_indexunary, indexunary_wrapper.cffi, ret_type._carg, dtype._carg, dtype2._carg + ), + "IndexUnaryOp", + new_indexunary, + ) + op = TypedUserIndexUnaryOp( + self, + self.name, + dtype, + ret_type, + new_indexunary[0], + dtype2=dtype2, + ) + self._udt_types[dtypes] = ret_type + self._udt_ops[dtypes] = op + return op + + @classmethod + def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=False): + if parameterized: + return ParameterizedIndexUnaryOp(name, func, anonymous=True, is_udt=is_udt) + return cls._build(name, func, anonymous=True, is_udt=is_udt) + + @classmethod + def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=False): + module, funcname = cls._remove_nesting(name) + if lazy: + module._delayed[funcname] = ( + cls.register_new, + {"name": name, "func": func, "parameterized": parameterized}, + ) + elif parameterized: + indexunary_op = ParameterizedIndexUnaryOp(name, func, is_udt=is_udt) + setattr(module, funcname, indexunary_op) + else: + indexunary_op = cls._build(name, func, is_udt=is_udt) + setattr(module, funcname, indexunary_op) + # If return type is BOOL, register additionally as a SelectOp + if all(x == BOOL for x in indexunary_op.types.values()): + setattr(select, funcname, SelectOp._from_indexunary(indexunary_op)) + + if not cls._initialized: + _STANDARD_OPERATOR_NAMES.add(f"{cls._modname}.{name}") + if not lazy: + return indexunary_op + @classmethod def _initialize(cls): - super()._initialize(include_in_ops=False) + if cls._initialized: + return + super()._initialize() # Update type information to include UINT64 for positional ops for name in ("tril", "triu", "diag", "offdiag", "colle", "colgt", "rowle", "rowgt"): op = getattr(indexunary, name) @@ -1167,65 +1428,156 @@ def _initialize(cls): op.types[UINT64] = output_type op._typed_ops[UINT64] = typed_op op.coercions[UINT64] = INT64 + # Add SelectOp when it makes sense + for name in ( + "tril", + "triu", + "diag", + "offdiag", + "colle", + "colgt", + "rowle", + "rowgt", + "valueeq", + "valuene", + "valuegt", + "valuege", + "valuelt", + "valuele", + ): + iop = getattr(indexunary, name) + setattr(select, name, SelectOp._from_indexunary(iop)) cls._initialized = True def __init__( self, name, + func=None, *, anonymous=False, is_positional=False, + is_udt=False, + numba_func=None, ): super().__init__(name, anonymous=anonymous) + self.orig_func = func + self._numba_func = numba_func self.is_positional = is_positional + self._is_udt = is_udt + if is_udt: + self._udt_types = {} # {dtype: DataType} + self._udt_ops = {} # {dtype: TypedUserIndexUnaryOp} + + def __reduce__(self): + if self._anonymous: + if hasattr(self.orig_func, "_parameterized_info"): + return (_deserialize_parameterized, self.orig_func._parameterized_info) + return (self.register_anonymous, (self.orig_func, self.name)) + name = f"indexunary.{self.name}" + if name in _STANDARD_OPERATOR_NAMES: + return name + return (self._deserialize, (self.name, self.orig_func)) __call__ = TypedBuiltinIndexUnaryOp.__call__ class SelectOp(OpBase): - __slots__ = "is_positional" + __slots__ = "orig_func", "is_positional", "_is_udt", "_numba_func" _module = select _modname = "select" - _is_udt = False _custom_dtype = None _typed_class = TypedBuiltinSelectOp - _parse_config = { - "trim_from_front": 4, - "num_underscores": 1, - "re_exprs_return_bool": [ - re.compile("^GrB_(TRIL|TRIU|DIAG|OFFDIAG|COLLE|COLGT|ROWLE|ROWGT)$"), - re.compile( - "^GrB_(VALUEEQ|VALUENE|VALUEGT|VALUEGE|VALUELT|VALUELE)" - "_(BOOL|INT8|UINT8|INT16|UINT16|INT32|UINT32|INT64|UINT64|FP32|FP64)$" - ), - re.compile("^GxB_(VALUEEQ|VALUENE)_(FC32|FC64)$"), - ], - } - _positional = {"tril", "triu", "diag", "offdiag", "colle", "colgt", "rowle", "rowgt"} + _typed_user_class = TypedUserSelectOp + + @classmethod + def _from_indexunary(cls, iop): + obj = cls( + iop.name, + iop.orig_func, + anonymous=iop._anonymous, + is_positional=iop.is_positional, + is_udt=iop._is_udt, + numba_func=iop._numba_func, + ) + if not all(x == BOOL for x in iop.types.values()): + raise ValueError("SelectOp must have BOOL return type") + for type, t in iop._typed_ops.items(): + if iop.orig_func is not None: + op = cls._typed_user_class( + obj, + iop.name, + t.type, + t.return_type, + t.gb_obj, + ) + else: + op = cls._typed_class( + obj, + iop.name, + t.type, + t.return_type, + t.gb_obj, + t.gb_name, + ) + # type is not always equal to t.type, so can't use op._add + # but otherwise perform the same logic + obj._typed_ops[type] = op + obj.types[type] = op.return_type + return obj + + @classmethod + def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=False): + if parameterized: + return ParameterizedSelectOp(name, func, anonymous=True, is_udt=is_udt) + iop = IndexUnaryOp._build(name, func, anonymous=True, is_udt=is_udt) + return SelectOp._from_indexunary(iop) + + @classmethod + def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=False): + iop = IndexUnaryOp.register_new( + name, func, parameterized=parameterized, is_udt=is_udt, lazy=lazy + ) + if not all(x == BOOL for x in iop.types.values()): + raise ValueError("SelectOp must have BOOL return type") + if lazy: + return getattr(select, iop.name) @classmethod def _initialize(cls): - super()._initialize(include_in_ops=False) - # Update type information to include UINT64 for positional ops - for name in ("tril", "triu", "diag", "offdiag", "colle", "colgt", "rowle", "rowgt"): - op = getattr(select, name) - typed_op = op._typed_ops[BOOL] - output_type = op.types[BOOL] - if UINT64 not in op.types: - op.types[UINT64] = output_type - op._typed_ops[UINT64] = typed_op - op.coercions[UINT64] = BOOL + if cls._initialized: + return + # IndexUnaryOp adds it boolean-returning objects to SelectOp + IndexUnaryOp._initialize() cls._initialized = True def __init__( self, name, + func=None, *, anonymous=False, is_positional=False, + is_udt=False, + numba_func=None, ): super().__init__(name, anonymous=anonymous) + self.orig_func = func + self._numba_func = numba_func self.is_positional = is_positional + self._is_udt = is_udt + if is_udt: + self._udt_types = {} # {dtype: DataType} + self._udt_ops = {} # {dtype: TypedUserIndexUnaryOp} + + def __reduce__(self): + if self._anonymous: + if hasattr(self.orig_func, "_parameterized_info"): + return (_deserialize_parameterized, self.orig_func._parameterized_info) + return (self.register_anonymous, (self.orig_func, self.name)) + name = f"select.{self.name}" + if name in _STANDARD_OPERATOR_NAMES: + return name + return (self._deserialize, (self.name, self.orig_func)) __call__ = TypedBuiltinSelectOp.__call__ @@ -1732,6 +2084,8 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal @classmethod def _initialize(cls): + if cls._initialized: + return super()._initialize() # Rename div to cdiv cdiv = binary.cdiv = op.cdiv = BinaryOp("cdiv") @@ -2087,6 +2441,8 @@ def _is_udt(self): @classmethod def _initialize(cls): + if cls._initialized: + return super()._initialize() lor = monoid.lor._typed_ops[BOOL] land = monoid.land._typed_ops[BOOL] @@ -2272,6 +2628,8 @@ def register_new(cls, name, monoid, binaryop, *, lazy=False): @classmethod def _initialize(cls): + if cls._initialized: + return super()._initialize() # Rename div to cdiv (truncate towards 0) div_semirings = { @@ -2693,6 +3051,10 @@ def get_semiring(monoid, binaryop, name=None): unary.register_new = UnaryOp.register_new unary.register_anonymous = UnaryOp.register_anonymous +indexunary.register_new = IndexUnaryOp.register_new +indexunary.register_anonymous = IndexUnaryOp.register_anonymous +select.register_new = SelectOp.register_new +select.register_anonymous = SelectOp.register_anonymous binary.register_new = BinaryOp.register_new binary.register_anonymous = BinaryOp.register_anonymous monoid.register_new = Monoid.register_new diff --git a/graphblas/select/__init__.py b/graphblas/select/__init__.py index 126ccf366..df839bad5 100644 --- a/graphblas/select/__init__.py +++ b/graphblas/select/__init__.py @@ -4,9 +4,6 @@ # boolean and they default to `select` when called, rather than # `apply`). _delayed = {} -from graphblas import operator # noqa isort:skip - -del operator def __dir__(): diff --git a/graphblas/semiring/__init__.py b/graphblas/semiring/__init__.py index d72c48e6b..30acdf87f 100644 --- a/graphblas/semiring/__init__.py +++ b/graphblas/semiring/__init__.py @@ -1,11 +1,8 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of Semiring instances _delayed = {} -from graphblas import operator # noqa isort:skip from . import numpy # noqa isort:skip -del operator - def __dir__(): return globals().keys() | _delayed.keys() diff --git a/graphblas/semiring/numpy.py b/graphblas/semiring/numpy.py index 4b632d89a..38d6c8c99 100644 --- a/graphblas/semiring/numpy.py +++ b/graphblas/semiring/numpy.py @@ -7,9 +7,9 @@ """ import itertools as _itertools +from .. import _STANDARD_OPERATOR_NAMES from .. import binary as _binary from .. import monoid as _monoid -from .. import operator as _operator from ..binary.numpy import _binary_names from ..monoid.numpy import _monoid_identities @@ -83,7 +83,7 @@ }, ) } -_operator._STANDARD_OPERATOR_NAMES.update(f"semiring.numpy.{name}" for name in _semiring_names) +_STANDARD_OPERATOR_NAMES.update(f"semiring.numpy.{name}" for name in _semiring_names) __all__ = list(_semiring_names) @@ -92,6 +92,8 @@ def __dir__(): def __getattr__(name): + from .. import operator as _operator + if name in _delayed: func, kwargs = _delayed.pop(name) if type(kwargs["binaryop"]) is str: diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index e3dc3bed1..41ffd2fa1 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -4,7 +4,19 @@ import pytest import graphblas as gb -from graphblas import agg, binary, dtypes, lib, monoid, op, operator, semiring, unary +from graphblas import ( + agg, + binary, + dtypes, + indexunary, + lib, + monoid, + op, + operator, + select, + semiring, + unary, +) from graphblas.dtypes import ( BOOL, FP32, @@ -1134,6 +1146,11 @@ def _udt_first(x, y): expected = expected.apply(gb.unary.positioni).new() assert result.isequal(expected) + result = indexunary.rowindex(v).new() + assert result.isequal(Vector.from_values([0, 1, 2], [0, 1, 2])) + result = select.rowle(v, 2).new() + assert result.isequal(v) + class BreakCompile: pass diff --git a/graphblas/unary/__init__.py b/graphblas/unary/__init__.py index 75293e1a0..b95d28e00 100644 --- a/graphblas/unary/__init__.py +++ b/graphblas/unary/__init__.py @@ -1,11 +1,8 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of UnaryOp instances _delayed = {} -from graphblas import operator # noqa isort:skip from . import numpy # noqa isort:skip -del operator - def __dir__(): return globals().keys() | _delayed.keys() diff --git a/graphblas/unary/numpy.py b/graphblas/unary/numpy.py index 9a910cf1d..5f0e0f836 100644 --- a/graphblas/unary/numpy.py +++ b/graphblas/unary/numpy.py @@ -7,8 +7,8 @@ """ import numpy as _np +from .. import _STANDARD_OPERATOR_NAMES from .. import config as _config -from .. import operator as _operator from .. import unary as _unary from ..dtypes import _supports_complex @@ -108,7 +108,7 @@ _numpy_to_graphblas["conjugate"] = "conj" # _graphblas_to_numpy = {val: key for key, val in _numpy_to_graphblas.items()} # Soon... -_operator._STANDARD_OPERATOR_NAMES.update(f"unary.numpy.{name}" for name in _unary_names) +_STANDARD_OPERATOR_NAMES.update(f"unary.numpy.{name}" for name in _unary_names) __all__ = list(_unary_names) @@ -127,6 +127,8 @@ def __getattr__(name): if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_unary, _numpy_to_graphblas[name]) else: + from .. import operator as _operator + numpy_func = getattr(_np, name) _operator.UnaryOp.register_new(f"numpy.{name}", lambda x: numpy_func(x)) if name == "reciprocal": From cf56d6a8f037212488a17bffb626bf662a514575 Mon Sep 17 00:00:00 2001 From: Jim Kitchen Date: Mon, 16 May 2022 10:38:33 -0500 Subject: [PATCH 02/11] Tests for IndexUnary UDF --- graphblas/matrix.py | 2 +- graphblas/tests/test_matrix.py | 38 ++++++++++++++++++++++++++++++++++ graphblas/tests/test_vector.py | 29 ++++++++++++++++++++++++++ graphblas/vector.py | 2 +- 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/graphblas/matrix.py b/graphblas/matrix.py index 7e5607dea..e24a1145c 100644 --- a/graphblas/matrix.py +++ b/graphblas/matrix.py @@ -807,7 +807,7 @@ def apply(self, op, right=None, *, left=None): expr_repr = "{1.name}.apply({op}, left={0._expr_name})" elif left is None: if type(right) is not Scalar: - dtype = self.dtype if self.dtype._is_udt else None + dtype = self.dtype if (self.dtype._is_udt and not op.is_positional) else None try: right = Scalar.from_value(right, dtype, is_cscalar=None, name="") except TypeError: diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index a73c10cd0..34669d99c 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -1139,6 +1139,44 @@ def test_select(A): assert w7.isequal(Aupper) +def test_indexunary_udf(A): + def threex_minusthunk(x, row, col, thunk): + return 3 * x - thunk + + indexunary.register_new("threex_minusthunk", threex_minusthunk) + assert hasattr(indexunary, "threex_minusthunk") + assert not hasattr(select, "threex_minusthunk") + with pytest.raises(ValueError): + select.register_anonymous(threex_minusthunk) + expected = Matrix.from_values( + [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], + [0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6], + [5, 2, 5, -1, 11, 5, 17, 20, 5, -1, 17, 8]) + result = indexunary.threex_minusthunk(A, 4).new() + assert result.isequal(expected) + + def iii(x, row, col, thunk): + return (row + col) // 2 >= thunk + + indexunary.register_new("iii", iii) + assert hasattr(indexunary, "iii") + assert hasattr(select, "iii") + iii_apply = indexunary.register_anonymous(iii) + expected = Matrix.from_values( + [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], + [0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6], + [False, False, True, True, True, False, True, True, True, True, True, True]) + result = iii_apply(A, 2).new() + assert result.isequal(expected) + iii_select = select.register_anonymous(iii) + expected = Matrix.from_values( + [3, 5, 6, 6, 1, 6, 2, 4, 1], + [2, 2, 2, 3, 4, 4, 5, 5, 6], + [3, 1, 5, 7, 8, 3, 1, 7, 4]) + result = iii_select(A, 2).new() + assert result.isequal(expected) + + def test_reduce_row(A): result = Vector.from_values([0, 1, 2, 3, 4, 5, 6], [5, 12, 1, 6, 7, 1, 15]) w = A.reduce_rowwise(monoid.plus).new() diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 1b7fdbac4..0352f2213 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -686,6 +686,35 @@ def test_select(v): assert w6.isequal(result) +def test_indexunary_udf(v): + def twox_minusthunk(x, row, col, thunk): + return 2 * x - thunk + + indexunary.register_new("twox_minusthunk", twox_minusthunk) + assert hasattr(indexunary, "twox_minusthunk") + assert not hasattr(select, "twox_minusthunk") + with pytest.raises(ValueError): + select.register_anonymous(twox_minusthunk) + expected = Vector.from_values([1, 3, 4, 6], [-2, -2, 0, -4], size=7) + result = indexunary.twox_minusthunk(v, 4).new() + assert result.isequal(expected) + + def ii(x, row, col, thunk): + return row // 2 >= thunk + + indexunary.register_new("ii", ii) + assert hasattr(indexunary, "ii") + assert hasattr(select, "ii") + ii_apply = indexunary.register_anonymous(ii) + expected = Vector.from_values([1, 3, 4, 6], [False, False, True, True], size=7) + result = ii_apply(v, 2).new() + assert result.isequal(expected) + ii_select = select.register_anonymous(ii) + expected = Vector.from_values([4, 6], [2, 0], size=7) + result = ii_select(v, 2).new() + assert result.isequal(expected) + + def test_reduce(v): s = v.reduce(monoid.plus).new() assert s == 4 diff --git a/graphblas/vector.py b/graphblas/vector.py index 2dfaeb351..46226c7f3 100644 --- a/graphblas/vector.py +++ b/graphblas/vector.py @@ -724,7 +724,7 @@ def apply(self, op, right=None, *, left=None): expr_repr = "{1.name}.apply({op}, left={0._expr_name})" elif left is None: if type(right) is not Scalar: - dtype = self.dtype if self.dtype._is_udt else None + dtype = self.dtype if (self.dtype._is_udt and not op.is_positional) else None try: right = Scalar.from_value(right, dtype, is_cscalar=None, name="") except TypeError: From bdd8cf77e0c58346841fcda289d2d7d3a9f83f75 Mon Sep 17 00:00:00 2001 From: Jim Kitchen Date: Mon, 16 May 2022 11:06:20 -0500 Subject: [PATCH 03/11] Add "index" alias Avoids the hard-to-read `vector.apply("row"...` stuff --- graphblas/operator.py | 31 +++++++++++++------------------ graphblas/select/__init__.py | 18 ++++++++++++++++-- graphblas/tests/test_matrix.py | 11 ++++++----- graphblas/tests/test_vector.py | 14 +++++++------- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/graphblas/operator.py b/graphblas/operator.py index 782b95297..6511f82e9 100644 --- a/graphblas/operator.py +++ b/graphblas/operator.py @@ -1428,25 +1428,18 @@ def _initialize(cls): op.types[UINT64] = output_type op._typed_ops[UINT64] = typed_op op.coercions[UINT64] = INT64 + # Add index->row alias to make it more intuitive which to use for vectors + indexunary.indexle = indexunary.rowle + indexunary.indexgt = indexunary.rowgt + indexunary.index = indexunary.rowindex + # fmt: off # Add SelectOp when it makes sense - for name in ( - "tril", - "triu", - "diag", - "offdiag", - "colle", - "colgt", - "rowle", - "rowgt", - "valueeq", - "valuene", - "valuegt", - "valuege", - "valuelt", - "valuele", - ): + for name in ("tril", "triu", "diag", "offdiag", + "colle", "colgt", "rowle", "rowgt", "indexle", "indexgt", + "valueeq", "valuene", "valuegt", "valuege", "valuelt", "valuele"): iop = getattr(indexunary, name) setattr(select, name, SelectOp._from_indexunary(iop)) + # fmt: on cls._initialized = True def __init__( @@ -3077,6 +3070,8 @@ def get_semiring(monoid, binaryop, name=None): "col>": select.colgt, "row<=": select.rowle, "row>": select.rowgt, + "index<=": select.indexle, + "index>": select.indexgt, } _str_to_binary = { "<": binary.lt, @@ -3196,13 +3191,13 @@ def semiring_from_string(string): def op_from_string(string): for func in [ + # Note: order matters here unary_from_string, binary_from_string, monoid_from_string, semiring_from_string, - # Give binary precedence over this - select_from_string, indexunary_from_string, + select_from_string, ]: try: return func(string) diff --git a/graphblas/select/__init__.py b/graphblas/select/__init__.py index df839bad5..8501e3d38 100644 --- a/graphblas/select/__init__.py +++ b/graphblas/select/__init__.py @@ -65,7 +65,7 @@ def value(expr): def row(expr): """ An advanced select method which allows for easily expressing - row index comparison logic. + Matrix row index comparison logic. Example usage: >>> gb.select.row(A <= 5) @@ -79,7 +79,7 @@ def row(expr): def column(expr): """ An advanced select method which allows for easily expressing - column index comparison logic. + Matrix column index comparison logic. Example usage: >>> gb.select.column(A <= 5) @@ -88,3 +88,17 @@ def column(expr): while being potentially nicer to read. """ return _resolve_expr(expr, "column", "col") + + +def index(expr): + """ + An advanced select method which allows for easily expressing + Vector index comparison logic. + + Example usage: + >>> gb.select.index(v <= 5) + + The example will dispatch to `gb.select.indexle(v, 5)` + while being potentially nicer to read. + """ + return _resolve_expr(expr, "index", "index") diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index 34669d99c..bd0ab0630 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -1151,7 +1151,8 @@ def threex_minusthunk(x, row, col, thunk): expected = Matrix.from_values( [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], [0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6], - [5, 2, 5, -1, 11, 5, 17, 20, 5, -1, 17, 8]) + [5, 2, 5, -1, 11, 5, 17, 20, 5, -1, 17, 8], + ) result = indexunary.threex_minusthunk(A, 4).new() assert result.isequal(expected) @@ -1165,14 +1166,14 @@ def iii(x, row, col, thunk): expected = Matrix.from_values( [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], [0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6], - [False, False, True, True, True, False, True, True, True, True, True, True]) + [False, False, True, True, True, False, True, True, True, True, True, True], + ) result = iii_apply(A, 2).new() assert result.isequal(expected) iii_select = select.register_anonymous(iii) expected = Matrix.from_values( - [3, 5, 6, 6, 1, 6, 2, 4, 1], - [2, 2, 2, 3, 4, 4, 5, 5, 6], - [3, 1, 5, 7, 8, 3, 1, 7, 4]) + [3, 5, 6, 6, 1, 6, 2, 4, 1], [2, 2, 2, 3, 4, 4, 5, 5, 6], [3, 1, 5, 7, 8, 3, 1, 7, 4] + ) result = iii_select(A, 2).new() assert result.isequal(expected) diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 0352f2213..f3f5f9da9 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -650,9 +650,9 @@ def test_apply_empty(v): def test_apply_indexunary(v): idx = [1, 3, 4, 6] vr = Vector.from_values(idx, idx) - r1 = v.apply(indexunary.rowindex).new() - r2 = v.apply("rowindex").new() - r3 = indexunary.rowindex(v).new() + r1 = v.apply(indexunary.index).new() + r2 = v.apply("index").new() + r3 = indexunary.index(v).new() assert r1.isequal(vr) assert r2.isequal(vr) assert r3.isequal(vr) @@ -675,9 +675,9 @@ def test_select(v): w1 = v.select(select.valueeq, 1).new() w2 = v.select("==", 1).new() w3 = select.rowle(v, 3).new() - w4 = v.select("row<=", 3).new() + w4 = v.select("index<=", 3).new() w5 = select.value(v == 1).new() - w6 = select.row(v < 4).new() + w6 = select.index(v < 4).new() assert w1.isequal(result) assert w2.isequal(result) assert w3.isequal(result) @@ -699,8 +699,8 @@ def twox_minusthunk(x, row, col, thunk): result = indexunary.twox_minusthunk(v, 4).new() assert result.isequal(expected) - def ii(x, row, col, thunk): - return row // 2 >= thunk + def ii(x, idx, _, thunk): + return idx // 2 >= thunk indexunary.register_new("ii", ii) assert hasattr(indexunary, "ii") From 611d90badb8d6b6d399de748b2e02b068620a5db Mon Sep 17 00:00:00 2001 From: Jim Kitchen Date: Mon, 16 May 2022 12:46:29 -0500 Subject: [PATCH 04/11] Fix broken test --- graphblas/tests/test_matrix.py | 3 +++ graphblas/tests/test_op.py | 19 +++++++++++++++++-- graphblas/tests/test_vector.py | 3 +++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index bd0ab0630..207a95390 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -1155,6 +1155,7 @@ def threex_minusthunk(x, row, col, thunk): ) result = indexunary.threex_minusthunk(A, 4).new() assert result.isequal(expected) + delattr(indexunary, "threex_minusthunk") def iii(x, row, col, thunk): return (row + col) // 2 >= thunk @@ -1176,6 +1177,8 @@ def iii(x, row, col, thunk): ) result = iii_select(A, 2).new() assert result.isequal(expected) + delattr(indexunary, "iii") + delattr(select, "iii") def test_reduce_row(A): diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index 41ffd2fa1..3b60639aa 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -676,11 +676,12 @@ def test_op_namespace(): # Make sure all have been initialized so `vars` below works for key in list(op._delayed): # pragma: no cover getattr(op, key) + known_aliases = {"index", "indexle", "indexgt"} opnames = { key for key, val in vars(op).items() if isinstance(val, (operator.OpBase, operator.ParameterizedUdf)) - } + } | known_aliases unarynames = { key for key, val in vars(unary).items() @@ -701,11 +702,25 @@ def test_op_namespace(): for key, val in vars(semiring).items() if isinstance(val, (operator.OpBase, operator.ParameterizedUdf)) } + indexunarynames = { + key + for key, val in vars(indexunary).items() + if isinstance(val, (operator.OpBase, operator.ParameterizedUdf)) + } + selectnames = { + key + for key, val in vars(select).items() + if isinstance(val, (operator.OpBase, operator.ParameterizedUdf)) + } assert not unarynames - opnames, unarynames - opnames assert not binarynames - opnames, binarynames - opnames assert not monoidnames - opnames, monoidnames - opnames assert not semiringnames - opnames, semiringnames - opnames - assert not opnames - (unarynames | binarynames | monoidnames | semiringnames) + assert not indexunarynames - opnames, indexunarynames - opnames + assert not selectnames - opnames, selectnames - opnames + assert not opnames - ( + unarynames | binarynames | monoidnames | semiringnames | indexunarynames | selectnames + ) @pytest.mark.slow diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index f3f5f9da9..61070fe1e 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -698,6 +698,7 @@ def twox_minusthunk(x, row, col, thunk): expected = Vector.from_values([1, 3, 4, 6], [-2, -2, 0, -4], size=7) result = indexunary.twox_minusthunk(v, 4).new() assert result.isequal(expected) + delattr(indexunary, "twox_minusthunk") def ii(x, idx, _, thunk): return idx // 2 >= thunk @@ -713,6 +714,8 @@ def ii(x, idx, _, thunk): expected = Vector.from_values([4, 6], [2, 0], size=7) result = ii_select(v, 2).new() assert result.isequal(expected) + delattr(indexunary, "ii") + delattr(select, "ii") def test_reduce(v): From 0833477b8fbb3ac4f2d2d517ff1e8fc2768110b6 Mon Sep 17 00:00:00 2001 From: Jim Kitchen Date: Mon, 16 May 2022 13:05:12 -0500 Subject: [PATCH 05/11] Fix imports --- graphblas/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/graphblas/__init__.py b/graphblas/__init__.py index 4a6db6de3..036459318 100644 --- a/graphblas/__init__.py +++ b/graphblas/__init__.py @@ -163,12 +163,19 @@ def _init(backend_arg, blocking, automatic=False): "graphblas._agg", "graphblas.agg", "graphblas.base", + "graphblas.binary", + "graphblas.indexunary", "graphblas.io", "graphblas.matrix", - "graphblas.scalar", - "graphblas.vector", + "graphblas.monoid", + "graphblas.op", "graphblas.recorder", + "graphblas.scalar", + "graphblas.select", + "graphblas.semiring", "graphblas.ss", + "graphblas.unary", + "graphblas.vector", } From 7584b2bf1bfdf535fb96b7693731efbdf8cca781 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 16 May 2022 15:32:55 -0500 Subject: [PATCH 06/11] Fix import issues and slightly untangle "needs operator" things --- .github/workflows/test_and_build.yml | 5 +---- graphblas/__init__.py | 25 ++++++++-------------- graphblas/binary/__init__.py | 5 +++++ graphblas/indexunary/__init__.py | 5 +++++ graphblas/monoid/__init__.py | 5 +++++ graphblas/op/__init__.py | 5 +++++ graphblas/operator.py | 2 +- graphblas/select/__init__.py | 5 +++++ graphblas/semiring/__init__.py | 5 +++++ graphblas/unary/__init__.py | 5 +++++ scripts/test_imports.sh | 31 ++++++++++++++++++++++++---- 11 files changed, 73 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 281be38d8..c581c04aa 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -91,11 +91,8 @@ jobs: # conda install -c conda-forge pygraphblas - name: Verify build run: | - # Make sure `from graphblas import *` works as expected - python -c "from graphblas import * ; Matrix" # Make sure all top-level imports work - ( for attr in Matrix Scalar Vector Recorder _agg agg base binary descriptor dtypes exceptions expr ffi formatting infix init io lib mask matrix monoid op operator scalar semiring tests unary vector recorder _ss ss ; do echo python -c \"from graphblas import $attr\" ; if ! python -c "from graphblas import $attr" ; then exit 1 ; fi ; done ) - ( for attr in _agg agg base binary descriptor dtypes exceptions expr formatting infix io mask matrix monoid op operator scalar semiring tests unary vector recorder _ss ss ; do echo python -c \"import graphblas.$attr\" ; if ! python -c "import graphblas.$attr" ; then exit 1 ; fi ; done ) + ./scripts/test_imports.sh - name: Unit tests # if: (! contains(matrix.cfg.testopts, 'pygraphblas')) || (matrix.cfg.pyver != 3.9) run: | diff --git a/graphblas/__init__.py b/graphblas/__init__.py index 036459318..245e00030 100644 --- a/graphblas/__init__.py +++ b/graphblas/__init__.py @@ -163,19 +163,9 @@ def _init(backend_arg, blocking, automatic=False): "graphblas._agg", "graphblas.agg", "graphblas.base", - "graphblas.binary", - "graphblas.indexunary", - "graphblas.io", - "graphblas.matrix", "graphblas.monoid", - "graphblas.op", - "graphblas.recorder", "graphblas.scalar", - "graphblas.select", "graphblas.semiring", - "graphblas.ss", - "graphblas.unary", - "graphblas.vector", } @@ -199,14 +189,17 @@ def _load(name): class _GraphblasModuleFinder(_MetaPathFinder): def find_spec(self, fullname, path, target=None): + if not fullname.startswith("graphblas."): + return if fullname in _NEEDS_OPERATOR and "operator" not in globals(): _load("operator") - name = fullname[7:] # Trim "graphblas." - if name in globals(): # pragma: no cover - # Make sure we execute the module only once - module = globals()[name] - spec = module.__spec__ - spec.loader = _SkipLoad(module, spec.loader) + name = fullname.split(".")[1] + if name in globals(): + # Make sure we execute the module only once + module = globals()[name] + spec = module.__spec__ + spec.loader = _SkipLoad(module, spec.loader) + if fullname == f"graphblas.{name}": return spec diff --git a/graphblas/binary/__init__.py b/graphblas/binary/__init__.py index 899d6c7a5..d792c2867 100644 --- a/graphblas/binary/__init__.py +++ b/graphblas/binary/__init__.py @@ -29,3 +29,8 @@ def __getattr__(key): rv._commutes_to = other return rv raise AttributeError(f"module {__name__!r} has no attribute {key!r}") + + +from .. import operator # noqa isort:skip + +del operator diff --git a/graphblas/indexunary/__init__.py b/graphblas/indexunary/__init__.py index fd8611158..f0b4154ed 100644 --- a/graphblas/indexunary/__init__.py +++ b/graphblas/indexunary/__init__.py @@ -14,3 +14,8 @@ def __getattr__(key): globals()[key] = rv return rv raise AttributeError(f"module {__name__!r} has no attribute {key!r}") + + +from .. import operator # noqa isort:skip + +del operator diff --git a/graphblas/monoid/__init__.py b/graphblas/monoid/__init__.py index 9181c7ac5..2cd493e48 100644 --- a/graphblas/monoid/__init__.py +++ b/graphblas/monoid/__init__.py @@ -19,3 +19,8 @@ def __getattr__(key): globals()[key] = rv return rv raise AttributeError(f"module {__name__!r} has no attribute {key!r}") + + +from .. import operator # noqa isort:skip + +del operator diff --git a/graphblas/op/__init__.py b/graphblas/op/__init__.py index b214e4b58..8b8547685 100644 --- a/graphblas/op/__init__.py +++ b/graphblas/op/__init__.py @@ -15,3 +15,8 @@ def __getattr__(key): globals()[key] = rv return rv raise AttributeError(f"module {__name__!r} has no attribute {key!r}") + + +from .. import operator # noqa isort:skip + +del operator diff --git a/graphblas/operator.py b/graphblas/operator.py index 6511f82e9..27afd3a27 100644 --- a/graphblas/operator.py +++ b/graphblas/operator.py @@ -34,7 +34,7 @@ if _supports_complex: from .dtypes import FC32, FC64 -from . import binary, indexunary, monoid, op, select, semiring, unary # noqa isort:skip +from . import binary, indexunary, monoid, op, select, semiring, unary # isort:skip ffi_new = ffi.new UNKNOWN_OPCLASS = "UnknownOpClass" diff --git a/graphblas/select/__init__.py b/graphblas/select/__init__.py index 8501e3d38..1d925b14e 100644 --- a/graphblas/select/__init__.py +++ b/graphblas/select/__init__.py @@ -102,3 +102,8 @@ def index(expr): while being potentially nicer to read. """ return _resolve_expr(expr, "index", "index") + + +from .. import operator # noqa isort:skip + +del operator diff --git a/graphblas/semiring/__init__.py b/graphblas/semiring/__init__.py index 30acdf87f..f022f798a 100644 --- a/graphblas/semiring/__init__.py +++ b/graphblas/semiring/__init__.py @@ -23,3 +23,8 @@ def __getattr__(key): globals()[key] = rv return rv raise AttributeError(f"module {__name__!r} has no attribute {key!r}") + + +from .. import operator # noqa isort:skip + +del operator diff --git a/graphblas/unary/__init__.py b/graphblas/unary/__init__.py index b95d28e00..9aaaa5357 100644 --- a/graphblas/unary/__init__.py +++ b/graphblas/unary/__init__.py @@ -15,3 +15,8 @@ def __getattr__(key): globals()[key] = rv return rv raise AttributeError(f"module {__name__!r} has no attribute {key!r}") + + +from .. import operator # noqa isort:skip + +del operator diff --git a/scripts/test_imports.sh b/scripts/test_imports.sh index 033a3e04d..7104d8fb4 100755 --- a/scripts/test_imports.sh +++ b/scripts/test_imports.sh @@ -1,7 +1,30 @@ #!/usr/bin/env bash # # Make sure imports work. Also, this is a good way to measure import performance. -(for attr in Matrix Scalar Vector Recorder _agg agg base binary descriptor \ +if ! python -c "from graphblas import * ; Matrix" ; then exit 1 ; fi +if ! python -c "from graphblas.binary import plus" ; then exit 1 ; fi +if ! python -c "from graphblas.indexunary import tril" ; then exit 1 ; fi +if ! python -c "from graphblas.monoid import plus" ; then exit 1 ; fi +if ! python -c "from graphblas.op import plus" ; then exit 1 ; fi +if ! python -c "from graphblas.select import tril" ; then exit 1 ; fi +if ! python -c "from graphblas.semiring import plus_times" ; then exit 1 ; fi +if ! python -c "from graphblas.unary import exp" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.binary.plus" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.indexunary.tril" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.monoid.plus" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.op.plus" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.select.tril" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.semiring.plus_times" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.unary.exp" ; then exit 1 ; fi +if ! python -c "from graphblas import binary ; binary.plus" ; then exit 1 ; fi +if ! python -c "from graphblas import indexunary ; indexunary.tril" ; then exit 1 ; fi +if ! python -c "from graphblas import monoid ; monoid.plus" ; then exit 1 ; fi +if ! python -c "from graphblas import op ; op.plus" ; then exit 1 ; fi +if ! python -c "from graphblas import select ; select.tril" ; then exit 1 ; fi +if ! python -c "from graphblas import semiring ; semiring.plus_times" ; then exit 1 ; fi +if ! python -c "from graphblas import unary ; unary.exp" ; then exit 1 ; fi + +if ! (for attr in Matrix Scalar Vector Recorder _agg agg base binary descriptor \ dtypes exceptions expr ffi formatting infix init io lib mask matrix monoid \ op operator scalar select semiring tests unary vector recorder _ss ss do echo python -c \"from graphblas import $attr\" @@ -9,8 +32,8 @@ then exit 1 fi done -) -(for attr in _agg agg base binary descriptor dtypes exceptions \ +) ; then exit 1 ; fi +if ! (for attr in _agg agg base binary descriptor dtypes exceptions \ expr formatting infix io mask matrix monoid op operator scalar \ select semiring tests unary vector recorder _ss ss do echo python -c \"import graphblas.$attr\" @@ -18,4 +41,4 @@ then exit 1 fi done -) +) ; then exit 1 ; fi From a2c6b748f0e6089c5db4bcd2b255d95f4f940b40 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 16 May 2022 15:37:54 -0500 Subject: [PATCH 07/11] Don't rename operator to _operator inside func --- graphblas/binary/numpy.py | 4 ++-- graphblas/monoid/numpy.py | 4 ++-- graphblas/semiring/numpy.py | 4 ++-- graphblas/unary/numpy.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/graphblas/binary/numpy.py b/graphblas/binary/numpy.py index 5740611ea..c301b5640 100644 --- a/graphblas/binary/numpy.py +++ b/graphblas/binary/numpy.py @@ -140,10 +140,10 @@ def __getattr__(name): if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_binary, _numpy_to_graphblas[name]) else: - from .. import operator as _operator + from .. import operator numpy_func = getattr(_np, name) - _operator.BinaryOp.register_new(f"numpy.{name}", lambda x, y: numpy_func(x, y)) + operator.BinaryOp.register_new(f"numpy.{name}", lambda x, y: numpy_func(x, y)) rv = globals()[name] if name in _commutative: rv._commutes_to = rv diff --git a/graphblas/monoid/numpy.py b/graphblas/monoid/numpy.py index 3f2513580..0729420ae 100644 --- a/graphblas/monoid/numpy.py +++ b/graphblas/monoid/numpy.py @@ -136,8 +136,8 @@ def __getattr__(name): if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_monoid, _numpy_to_graphblas[name]) else: - from .. import operator as _operator + from .. import operator func = getattr(_binary.numpy, name) - _operator.Monoid.register_new(f"numpy.{name}", func, _monoid_identities[name]) + operator.Monoid.register_new(f"numpy.{name}", func, _monoid_identities[name]) return globals()[name] diff --git a/graphblas/semiring/numpy.py b/graphblas/semiring/numpy.py index 38d6c8c99..867af36ef 100644 --- a/graphblas/semiring/numpy.py +++ b/graphblas/semiring/numpy.py @@ -92,7 +92,7 @@ def __dir__(): def __getattr__(name): - from .. import operator as _operator + from .. import operator if name in _delayed: func, kwargs = _delayed.pop(name) @@ -117,7 +117,7 @@ def __getattr__(name): binary_name = "_".join(words[i:]) if hasattr(_binary.numpy, binary_name): # pragma: no branch break - _operator.get_semiring( + operator.get_semiring( getattr(_monoid.numpy, monoid_name), getattr(_binary.numpy, binary_name), name=f"numpy.{name}", diff --git a/graphblas/unary/numpy.py b/graphblas/unary/numpy.py index 5f0e0f836..0ba845b77 100644 --- a/graphblas/unary/numpy.py +++ b/graphblas/unary/numpy.py @@ -127,12 +127,12 @@ def __getattr__(name): if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_unary, _numpy_to_graphblas[name]) else: - from .. import operator as _operator + from .. import operator numpy_func = getattr(_np, name) - _operator.UnaryOp.register_new(f"numpy.{name}", lambda x: numpy_func(x)) + operator.UnaryOp.register_new(f"numpy.{name}", lambda x: numpy_func(x)) if name == "reciprocal": # numba doesn't match numpy here - op = _operator.UnaryOp.register_anonymous(lambda x: 1 if x else 0) + op = operator.UnaryOp.register_anonymous(lambda x: 1 if x else 0) globals()[name]._add(op["BOOL"]) return globals()[name] From 27b845b8c16f14dc4cbac953c9f57bbabeb2b225 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 16 May 2022 22:51:30 -0500 Subject: [PATCH 08/11] Untangle imports! No need for import loaders or metapath finders. --- graphblas/__init__.py | 53 ++--------------------------- graphblas/_agg.py | 8 +++-- graphblas/agg.py | 5 ++- graphblas/binary/__init__.py | 2 +- graphblas/monoid/__init__.py | 2 +- graphblas/op/__init__.py | 2 +- graphblas/operator.py | 34 ++++++++++++------ graphblas/semiring/__init__.py | 2 +- graphblas/tests/test_formatting.py | 5 +-- graphblas/tests/test_infix.py | 4 +-- graphblas/tests/test_matrix.py | 16 +++++++-- graphblas/tests/test_op.py | 4 +-- graphblas/tests/test_prefix_scan.py | 4 +-- graphblas/tests/test_resolving.py | 4 +-- graphblas/tests/test_scalar.py | 4 +-- graphblas/tests/test_vector.py | 16 +++++++-- graphblas/unary/__init__.py | 2 +- scripts/test_imports.sh | 14 +++++--- 18 files changed, 83 insertions(+), 98 deletions(-) diff --git a/graphblas/__init__.py b/graphblas/__init__.py index 245e00030..54c5db0bd 100644 --- a/graphblas/__init__.py +++ b/graphblas/__init__.py @@ -1,7 +1,5 @@ import sys as _sys from importlib import import_module as _import_module -from importlib.abc import Loader as _Loader -from importlib.abc import MetaPathFinder as _MetaPathFinder from . import mask # noqa from ._version import get_versions @@ -159,15 +157,6 @@ def _init(backend_arg, blocking, automatic=False): # Ideally this is in operator.py, but lives here to avoid circular references _STANDARD_OPERATOR_NAMES = set() -_NEEDS_OPERATOR = { - "graphblas._agg", - "graphblas.agg", - "graphblas.base", - "graphblas.monoid", - "graphblas.scalar", - "graphblas.semiring", -} - def _load(name): if name in {"Matrix", "Vector", "Scalar", "Recorder"}: @@ -175,47 +164,11 @@ def _load(name): if module_name not in globals(): _load(module_name) module = globals()[module_name] - val = getattr(module, name) - globals()[name] = val + globals()[name] = getattr(module, name) else: # Everything else is a module - module = _import_module(f".{name}", __name__) - globals()[name] = module - spec = module.__spec__ - if not isinstance(spec.loader, _SkipLoad): - # Make sure we execute the module only once - spec.loader = _SkipLoad(module, spec.loader) - - -class _GraphblasModuleFinder(_MetaPathFinder): - def find_spec(self, fullname, path, target=None): - if not fullname.startswith("graphblas."): - return - if fullname in _NEEDS_OPERATOR and "operator" not in globals(): - _load("operator") - name = fullname.split(".")[1] - if name in globals(): - # Make sure we execute the module only once - module = globals()[name] - spec = module.__spec__ - spec.loader = _SkipLoad(module, spec.loader) - if fullname == f"graphblas.{name}": - return spec - - -class _SkipLoad(_Loader): - def __init__(self, module, orig_loader): - self.module = module - self.orig_loader = orig_loader - - def create_module(self, spec): - return self.module - - def exec_module(self, module): - pass - - -_sys.meta_path.insert(0, _GraphblasModuleFinder()) + globals()[name] = _import_module(f".{name}", __name__) + __version__ = get_versions()["version"] del get_versions diff --git a/graphblas/_agg.py b/graphblas/_agg.py index 980778532..eb43c2abe 100644 --- a/graphblas/_agg.py +++ b/graphblas/_agg.py @@ -5,8 +5,6 @@ from . import agg, binary, monoid, semiring, unary from .dtypes import INT64, lookup_dtype -from .operator import get_typed_op -from .scalar import Scalar def _get_types(ops, initdtype): @@ -124,7 +122,7 @@ def _new(self, updater, expr, *, in_composite=False): if agg._monoid is not None: x = expr.args[0] method = getattr(x, expr.method_name) - if expr.output_type is Scalar: + if expr.output_type.__name__ == "Scalar": expr = method(agg._monoid[self.type], allow_empty=not expr._is_cscalar) else: expr = method(agg._monoid[self.type]) @@ -621,3 +619,7 @@ def _first_last_index(agg, updater, expr, *, in_composite, semiring): types=[semiring.min_secondi], any_dtype=INT64, ) +agg.Aggregator = Aggregator +agg.TypedAggregator = TypedAggregator + +from .operator import get_typed_op # noqa isort:skip diff --git a/graphblas/agg.py b/graphblas/agg.py index 469a5b23c..fb4dd4fa1 100644 --- a/graphblas/agg.py +++ b/graphblas/agg.py @@ -70,4 +70,7 @@ """ # All items are dynamically added by classes in _agg.py # This module acts as a container of Aggregator instances -from ._agg import Aggregator, TypedAggregator # noqa +from . import operator + +del operator +from . import _agg # noqa isort:skip diff --git a/graphblas/binary/__init__.py b/graphblas/binary/__init__.py index d792c2867..85a7e0867 100644 --- a/graphblas/binary/__init__.py +++ b/graphblas/binary/__init__.py @@ -8,7 +8,6 @@ "rfloordiv": "floordiv", "rpow": "pow", } -from . import numpy # noqa isort:skip def __dir__(): @@ -32,5 +31,6 @@ def __getattr__(key): from .. import operator # noqa isort:skip +from . import numpy # noqa isort:skip del operator diff --git a/graphblas/monoid/__init__.py b/graphblas/monoid/__init__.py index 2cd493e48..cf8118d5e 100644 --- a/graphblas/monoid/__init__.py +++ b/graphblas/monoid/__init__.py @@ -1,7 +1,6 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of Monoid instances _delayed = {} -from . import numpy # noqa isort:skip def __dir__(): @@ -22,5 +21,6 @@ def __getattr__(key): from .. import operator # noqa isort:skip +from . import numpy # noqa isort:skip del operator diff --git a/graphblas/op/__init__.py b/graphblas/op/__init__.py index 8b8547685..3e8ed888d 100644 --- a/graphblas/op/__init__.py +++ b/graphblas/op/__init__.py @@ -1,7 +1,6 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of all UnaryOp, BinaryOp, and Semiring instances _delayed = {} -from . import numpy # noqa isort:skip def __dir__(): @@ -18,5 +17,6 @@ def __getattr__(key): from .. import operator # noqa isort:skip +from . import numpy # noqa isort:skip del operator diff --git a/graphblas/operator.py b/graphblas/operator.py index 27afd3a27..b3dd425b5 100644 --- a/graphblas/operator.py +++ b/graphblas/operator.py @@ -9,7 +9,19 @@ import numba import numpy as np -from . import _STANDARD_OPERATOR_NAMES, config, ffi, lib +from . import ( + _STANDARD_OPERATOR_NAMES, + binary, + config, + ffi, + indexunary, + lib, + monoid, + op, + select, + semiring, + unary, +) from .dtypes import ( BOOL, FP32, @@ -34,8 +46,6 @@ if _supports_complex: from .dtypes import FC32, FC64 -from . import binary, indexunary, monoid, op, select, semiring, unary # isort:skip - ffi_new = ffi.new UNKNOWN_OPCLASS = "UnknownOpClass" @@ -2868,7 +2878,10 @@ def get_typed_op(op, dtype, dtype2=None, *, is_left_scalar=False, is_right_scala ) elif isinstance(op, TypedOpBase): return op - elif isinstance(op, Aggregator): + + from ._agg import Aggregator, TypedAggregator + + if isinstance(op, Aggregator): return op[dtype] elif isinstance(op, TypedAggregator): return op @@ -3214,14 +3227,11 @@ def op_from_string(string): semiring.from_string = semiring_from_string op.from_string = op_from_string -from . import agg # noqa isort:skip -from .agg import Aggregator, TypedAggregator # noqa isort:skip - _str_to_agg = { - "+": agg.sum, - "*": agg.prod, - "&": agg.all, - "|": agg.any, + "+": "sum", + "*": "prod", + "&": "all", + "|": "any", } @@ -3229,4 +3239,6 @@ def aggregator_from_string(string): return _from_string(string, agg, _str_to_agg, "sum[int]") +from . import agg # noqa isort:skip + agg.from_string = aggregator_from_string diff --git a/graphblas/semiring/__init__.py b/graphblas/semiring/__init__.py index f022f798a..0abeffbec 100644 --- a/graphblas/semiring/__init__.py +++ b/graphblas/semiring/__init__.py @@ -1,7 +1,6 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of Semiring instances _delayed = {} -from . import numpy # noqa isort:skip def __dir__(): @@ -26,5 +25,6 @@ def __getattr__(key): from .. import operator # noqa isort:skip +from . import numpy # noqa isort:skip del operator diff --git a/graphblas/tests/test_formatting.py b/graphblas/tests/test_formatting.py index 20801290f..1733839ef 100644 --- a/graphblas/tests/test_formatting.py +++ b/graphblas/tests/test_formatting.py @@ -2,14 +2,11 @@ import pytest import graphblas -from graphblas import dtypes, formatting, unary +from graphblas import Matrix, Scalar, Vector, dtypes, formatting, unary from graphblas.formatting import CSS_STYLE from .conftest import autocompute -from graphblas import Matrix, Scalar, Vector # isort:skip - - try: import pandas as pd except ImportError: # pragma: no cover diff --git a/graphblas/tests/test_infix.py b/graphblas/tests/test_infix.py index 74ae70725..d539aa99b 100644 --- a/graphblas/tests/test_infix.py +++ b/graphblas/tests/test_infix.py @@ -1,12 +1,10 @@ from pytest import fixture, raises -from graphblas import monoid, op +from graphblas import Matrix, Scalar, Vector, monoid, op from graphblas.exceptions import DimensionMismatch from .conftest import autocompute -from graphblas import Matrix, Scalar, Vector # isort:skip - @fixture def v1(): diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index 207a95390..9b9883476 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -9,7 +9,19 @@ from numpy.testing import assert_array_equal import graphblas -from graphblas import agg, binary, dtypes, indexunary, monoid, select, semiring, unary +from graphblas import ( + Matrix, + Scalar, + Vector, + agg, + binary, + dtypes, + indexunary, + monoid, + select, + semiring, + unary, +) from graphblas.exceptions import ( DimensionMismatch, EmptyObject, @@ -21,8 +33,6 @@ from .conftest import autocompute, compute -from graphblas import Matrix, Scalar, Vector # isort:skip - @pytest.fixture def A(): diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index 3b60639aa..ee0da366f 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -5,6 +5,8 @@ import graphblas as gb from graphblas import ( + Matrix, + Vector, agg, binary, dtypes, @@ -36,8 +38,6 @@ if dtypes._supports_complex: from graphblas.dtypes import FC32, FC64 -from graphblas import Matrix, Vector # isort:skip - def orig_types(op): return op.types.keys() - op.coercions.keys() diff --git a/graphblas/tests/test_prefix_scan.py b/graphblas/tests/test_prefix_scan.py index 3db493710..6bb4ec53b 100644 --- a/graphblas/tests/test_prefix_scan.py +++ b/graphblas/tests/test_prefix_scan.py @@ -2,9 +2,7 @@ import pytest import graphblas as gb -from graphblas import binary, monoid - -from graphblas import Matrix, Vector # isort:skip +from graphblas import Matrix, Vector, binary, monoid try: # gb.io.to_numpy currently requires scipy diff --git a/graphblas/tests/test_resolving.py b/graphblas/tests/test_resolving.py index 0a17b7f23..db84fc288 100644 --- a/graphblas/tests/test_resolving.py +++ b/graphblas/tests/test_resolving.py @@ -1,11 +1,9 @@ import numpy as np import pytest -from graphblas import binary, dtypes, replace, unary +from graphblas import Matrix, Scalar, Vector, binary, dtypes, replace, unary from graphblas.expr import Updater -from graphblas import Matrix, Scalar, Vector # isort:skip - def test_from_values_dtype_resolving(): u = Vector.from_values([0, 1, 2], [1, 2, 3], dtype=dtypes.INT32) diff --git a/graphblas/tests/test_scalar.py b/graphblas/tests/test_scalar.py index 4239f1c2a..97bebda8b 100644 --- a/graphblas/tests/test_scalar.py +++ b/graphblas/tests/test_scalar.py @@ -8,12 +8,10 @@ import pytest import graphblas as gb -from graphblas import binary, dtypes, replace +from graphblas import Matrix, Scalar, Vector, binary, dtypes, replace from .conftest import autocompute, compute -from graphblas import Matrix, Scalar, Vector # isort:skip - @pytest.fixture def s(): diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 61070fe1e..b91a06677 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -9,7 +9,19 @@ from numpy.testing import assert_array_equal import graphblas -from graphblas import agg, binary, dtypes, indexunary, monoid, select, semiring, unary +from graphblas import ( + Matrix, + Scalar, + Vector, + agg, + binary, + dtypes, + indexunary, + monoid, + select, + semiring, + unary, +) from graphblas.exceptions import ( DimensionMismatch, EmptyObject, @@ -20,8 +32,6 @@ from .conftest import autocompute, compute -from graphblas import Matrix, Scalar, Vector # isort:skip - @pytest.fixture def A(): diff --git a/graphblas/unary/__init__.py b/graphblas/unary/__init__.py index 9aaaa5357..c356a9b89 100644 --- a/graphblas/unary/__init__.py +++ b/graphblas/unary/__init__.py @@ -1,7 +1,6 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of UnaryOp instances _delayed = {} -from . import numpy # noqa isort:skip def __dir__(): @@ -18,5 +17,6 @@ def __getattr__(key): from .. import operator # noqa isort:skip +from . import numpy # noqa isort:skip del operator diff --git a/scripts/test_imports.sh b/scripts/test_imports.sh index 7104d8fb4..698aa99f3 100755 --- a/scripts/test_imports.sh +++ b/scripts/test_imports.sh @@ -2,6 +2,8 @@ # # Make sure imports work. Also, this is a good way to measure import performance. if ! python -c "from graphblas import * ; Matrix" ; then exit 1 ; fi +if ! python -c "from graphblas import agg, _agg" ; then exit 1 ; fi +if ! python -c "from graphblas.agg import count" ; then exit 1 ; fi if ! python -c "from graphblas.binary import plus" ; then exit 1 ; fi if ! python -c "from graphblas.indexunary import tril" ; then exit 1 ; fi if ! python -c "from graphblas.monoid import plus" ; then exit 1 ; fi @@ -9,6 +11,7 @@ if ! python -c "from graphblas.op import plus" ; then exit 1 ; fi if ! python -c "from graphblas.select import tril" ; then exit 1 ; fi if ! python -c "from graphblas.semiring import plus_times" ; then exit 1 ; fi if ! python -c "from graphblas.unary import exp" ; then exit 1 ; fi +if ! python -c "import graphblas as gb ; gb.agg.count" ; then exit 1 ; fi if ! python -c "import graphblas as gb ; gb.binary.plus" ; then exit 1 ; fi if ! python -c "import graphblas as gb ; gb.indexunary.tril" ; then exit 1 ; fi if ! python -c "import graphblas as gb ; gb.monoid.plus" ; then exit 1 ; fi @@ -16,6 +19,7 @@ if ! python -c "import graphblas as gb ; gb.op.plus" ; then exit 1 ; fi if ! python -c "import graphblas as gb ; gb.select.tril" ; then exit 1 ; fi if ! python -c "import graphblas as gb ; gb.semiring.plus_times" ; then exit 1 ; fi if ! python -c "import graphblas as gb ; gb.unary.exp" ; then exit 1 ; fi +if ! python -c "from graphblas import agg ; agg.count" ; then exit 1 ; fi if ! python -c "from graphblas import binary ; binary.plus" ; then exit 1 ; fi if ! python -c "from graphblas import indexunary ; indexunary.tril" ; then exit 1 ; fi if ! python -c "from graphblas import monoid ; monoid.plus" ; then exit 1 ; fi @@ -26,16 +30,18 @@ if ! python -c "from graphblas import unary ; unary.exp" ; then exit 1 ; fi if ! (for attr in Matrix Scalar Vector Recorder _agg agg base binary descriptor \ dtypes exceptions expr ffi formatting infix init io lib mask matrix monoid \ - op operator scalar select semiring tests unary vector recorder _ss ss + op operator scalar select semiring tests unary vector recorder _ss ss \ + _automethods _infixmethods _slice do echo python -c \"from graphblas import $attr\" if ! python -c "from graphblas import $attr" then exit 1 fi done ) ; then exit 1 ; fi -if ! (for attr in _agg agg base binary descriptor dtypes exceptions \ - expr formatting infix io mask matrix monoid op operator scalar \ - select semiring tests unary vector recorder _ss ss +if ! (for attr in _agg agg base binary binary.numpy descriptor dtypes exceptions \ + expr formatting infix io mask matrix monoid monoid.numpy op op.numpy operator scalar \ + select semiring semiring.numpy tests unary unary.numpy vector recorder _ss ss \ + _automethods _infixmethods _slice do echo python -c \"import graphblas.$attr\" if ! python -c "import graphblas.$attr" then exit 1 From c44addf923b192d087ceb78034c46e25d90a6fd8 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 16 May 2022 23:20:40 -0500 Subject: [PATCH 09/11] Move a couple imports to top-level of module --- graphblas/_ss/matrix.py | 7 +++---- graphblas/expr.py | 3 +-- graphblas/matrix.py | 15 +++++++++------ graphblas/recorder.py | 3 +-- graphblas/vector.py | 11 ++++++++--- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/graphblas/_ss/matrix.py b/graphblas/_ss/matrix.py index b29411264..266931b8f 100644 --- a/graphblas/_ss/matrix.py +++ b/graphblas/_ss/matrix.py @@ -3580,8 +3580,6 @@ def scan_columnwise(self, op=monoid.plus, *, name=None): ------- Vector """ - from .prefix_scan import prefix_scan - return prefix_scan(self._parent.T, op, name=name, within="scan_columnwise") def scan_rowwise(self, op=monoid.plus, *, name=None): @@ -3594,8 +3592,6 @@ def scan_rowwise(self, op=monoid.plus, *, name=None): ------- Vector """ - from .prefix_scan import prefix_scan - return prefix_scan(self._parent, op, name=name, within="scan_rowwise") def flatten(self, order="rowwise", *, name=None): @@ -4230,3 +4226,6 @@ def indptr_to_indices(indptr): # pragma: no cover for j in range(indptr[i], indptr[i + 1]): indices[j] = i return indices + + +from .prefix_scan import prefix_scan # noqa isort:skip diff --git a/graphblas/expr.py b/graphblas/expr.py index d00bda38a..ffa40ffd0 100644 --- a/graphblas/expr.py +++ b/graphblas/expr.py @@ -2,6 +2,7 @@ from . import lib, utils from .dtypes import _INDEX +from .mask import Mask from .utils import _CArray, output_type @@ -231,8 +232,6 @@ def parse_index(self, index, typ, size): try: index = list(index) except Exception: - from .mask import Mask - if isinstance(index, Mask): raise TypeError( f"Invalid type for index: {typ.__name__}.\n" diff --git a/graphblas/matrix.py b/graphblas/matrix.py index e24a1145c..831256a3e 100644 --- a/graphblas/matrix.py +++ b/graphblas/matrix.py @@ -11,7 +11,14 @@ from .expr import AmbiguousAssignOrExtract, IndexerResolver, Updater from .mask import StructuralMask, ValueMask from .operator import find_opclass, get_semiring, get_typed_op, op_from_string -from .scalar import _MATERIALIZE, Scalar, ScalarExpression, _as_scalar, _scalar_index +from .scalar import ( + _MATERIALIZE, + Scalar, + ScalarExpression, + ScalarIndexExpr, + _as_scalar, + _scalar_index, +) from .utils import ( _CArray, _Pointer, @@ -21,7 +28,7 @@ values_to_numpy_buffer, wrapdoc, ) -from .vector import Vector, VectorExpression +from .vector import Vector, VectorExpression, VectorIndexExpr ffi_new = ffi.new @@ -142,12 +149,8 @@ def __getitem__(self, keys): resolved_indexes = IndexerResolver(self, keys) shape = resolved_indexes.shape if not shape: - from .scalar import ScalarIndexExpr - return ScalarIndexExpr(self, resolved_indexes) elif len(shape) == 1: - from .vector import VectorIndexExpr - return VectorIndexExpr(self, resolved_indexes, *shape) else: return MatrixIndexExpr(self, resolved_indexes, *shape) diff --git a/graphblas/recorder.py b/graphblas/recorder.py index 605325510..610bff3cf 100644 --- a/graphblas/recorder.py +++ b/graphblas/recorder.py @@ -3,6 +3,7 @@ from . import base, lib from .base import _recorder from .dtypes import DataType +from .formatting import CSS_STYLE from .mask import Mask from .matrix import TransposedMatrix from .operator import TypedOpBase @@ -105,8 +106,6 @@ def is_recording(self): def _repr_markdown_(self): # Syntax highlighting from github-flavored markdown looks better than # using IPython.display.Code or pygments - from .formatting import CSS_STYLE - status = ( '
Date: Mon, 16 May 2022 23:37:02 -0500 Subject: [PATCH 10/11] Oh yeah, we did this for `dask-graphblas`. Undo! --- graphblas/tests/test_formatting.py | 5 ++++- graphblas/tests/test_infix.py | 4 +++- graphblas/tests/test_matrix.py | 16 +++------------- graphblas/tests/test_op.py | 4 ++-- graphblas/tests/test_prefix_scan.py | 4 +++- graphblas/tests/test_resolving.py | 4 +++- graphblas/tests/test_scalar.py | 4 +++- graphblas/tests/test_vector.py | 16 +++------------- 8 files changed, 24 insertions(+), 33 deletions(-) diff --git a/graphblas/tests/test_formatting.py b/graphblas/tests/test_formatting.py index 1733839ef..f46a6ef2e 100644 --- a/graphblas/tests/test_formatting.py +++ b/graphblas/tests/test_formatting.py @@ -2,11 +2,14 @@ import pytest import graphblas -from graphblas import Matrix, Scalar, Vector, dtypes, formatting, unary +from graphblas import dtypes, formatting, unary from graphblas.formatting import CSS_STYLE from .conftest import autocompute +from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) + + try: import pandas as pd except ImportError: # pragma: no cover diff --git a/graphblas/tests/test_infix.py b/graphblas/tests/test_infix.py index d539aa99b..3826cc445 100644 --- a/graphblas/tests/test_infix.py +++ b/graphblas/tests/test_infix.py @@ -1,10 +1,12 @@ from pytest import fixture, raises -from graphblas import Matrix, Scalar, Vector, monoid, op +from graphblas import monoid, op from graphblas.exceptions import DimensionMismatch from .conftest import autocompute +from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) + @fixture def v1(): diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index 9b9883476..16166f3fb 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -9,19 +9,7 @@ from numpy.testing import assert_array_equal import graphblas -from graphblas import ( - Matrix, - Scalar, - Vector, - agg, - binary, - dtypes, - indexunary, - monoid, - select, - semiring, - unary, -) +from graphblas import agg, binary, dtypes, indexunary, monoid, select, semiring, unary from graphblas.exceptions import ( DimensionMismatch, EmptyObject, @@ -33,6 +21,8 @@ from .conftest import autocompute, compute +from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) + @pytest.fixture def A(): diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index ee0da366f..512c47acb 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -5,8 +5,6 @@ import graphblas as gb from graphblas import ( - Matrix, - Vector, agg, binary, dtypes, @@ -38,6 +36,8 @@ if dtypes._supports_complex: from graphblas.dtypes import FC32, FC64 +from graphblas import Matrix, Vector # isort:skip (for dask-graphblas) + def orig_types(op): return op.types.keys() - op.coercions.keys() diff --git a/graphblas/tests/test_prefix_scan.py b/graphblas/tests/test_prefix_scan.py index 6bb4ec53b..d0f7e02d8 100644 --- a/graphblas/tests/test_prefix_scan.py +++ b/graphblas/tests/test_prefix_scan.py @@ -2,7 +2,9 @@ import pytest import graphblas as gb -from graphblas import Matrix, Vector, binary, monoid +from graphblas import binary, monoid + +from graphblas import Matrix, Vector # isort:skip (for dask-graphblas) try: # gb.io.to_numpy currently requires scipy diff --git a/graphblas/tests/test_resolving.py b/graphblas/tests/test_resolving.py index db84fc288..2f6724b53 100644 --- a/graphblas/tests/test_resolving.py +++ b/graphblas/tests/test_resolving.py @@ -1,9 +1,11 @@ import numpy as np import pytest -from graphblas import Matrix, Scalar, Vector, binary, dtypes, replace, unary +from graphblas import binary, dtypes, replace, unary from graphblas.expr import Updater +from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) + def test_from_values_dtype_resolving(): u = Vector.from_values([0, 1, 2], [1, 2, 3], dtype=dtypes.INT32) diff --git a/graphblas/tests/test_scalar.py b/graphblas/tests/test_scalar.py index 97bebda8b..6b659fc9a 100644 --- a/graphblas/tests/test_scalar.py +++ b/graphblas/tests/test_scalar.py @@ -8,10 +8,12 @@ import pytest import graphblas as gb -from graphblas import Matrix, Scalar, Vector, binary, dtypes, replace +from graphblas import binary, dtypes, replace from .conftest import autocompute, compute +from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) + @pytest.fixture def s(): diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index b91a06677..69229647d 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -9,19 +9,7 @@ from numpy.testing import assert_array_equal import graphblas -from graphblas import ( - Matrix, - Scalar, - Vector, - agg, - binary, - dtypes, - indexunary, - monoid, - select, - semiring, - unary, -) +from graphblas import agg, binary, dtypes, indexunary, monoid, select, semiring, unary from graphblas.exceptions import ( DimensionMismatch, EmptyObject, @@ -32,6 +20,8 @@ from .conftest import autocompute, compute +from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) + @pytest.fixture def A(): From 14e01f930296ae343d19dfeb49d0272c4016be29 Mon Sep 17 00:00:00 2001 From: Jim Kitchen Date: Tue, 17 May 2022 11:59:52 -0500 Subject: [PATCH 11/11] More test coverage --- graphblas/operator.py | 36 +++++++++++++++++----------------- graphblas/tests/test_matrix.py | 2 +- graphblas/tests/test_op.py | 13 +++++++++++- graphblas/tests/test_pickle.py | 8 ++++++++ graphblas/tests/test_vector.py | 2 +- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/graphblas/operator.py b/graphblas/operator.py index b3dd425b5..f31b3cc72 100644 --- a/graphblas/operator.py +++ b/graphblas/operator.py @@ -393,8 +393,8 @@ class TypedUserIndexUnaryOp(TypedOpBase): __slots__ = () opclass = "IndexUnaryOp" - def __init__(self, parent, name, type_, return_type, gb_obj): - super().__init__(parent, name, type_, return_type, gb_obj, f"{name}_{type_}") + def __init__(self, parent, name, type_, return_type, gb_obj, dtype2=None): + super().__init__(parent, name, type_, return_type, gb_obj, f"{name}_{type_}", dtype2=dtype2) @property def orig_func(self): @@ -1361,10 +1361,12 @@ def _compile_udt(self, dtype, dtype2): return self._udt_ops[dtypes] numba_func = self._numba_func - sig = (dtype.numba_type, dtype2.numba_type) + sig = (dtype.numba_type, UINT64.numba_type, UINT64.numba_type, dtype2.numba_type) numba_func.compile(sig) # Should we catch and give additional error message? ret_type = lookup_dtype(numba_func.overloads[sig].signature.return_type) - indexunary_wrapper, wrapper_sig = _get_udt_wrapper(numba_func, ret_type, dtype, dtype2) + indexunary_wrapper, wrapper_sig = _get_udt_wrapper( + numba_func, ret_type, dtype, dtype2, include_indexes=True + ) indexunary_wrapper = numba.cfunc(wrapper_sig, nopython=True)(indexunary_wrapper) new_indexunary = ffi_new("GrB_IndexUnaryOp*") @@ -1677,24 +1679,19 @@ def _pair_dtype(op, dtype, dtype2): return op[INT64] -def _get_udt_wrapper(numba_func, return_type, dtype, dtype2=None): +def _get_udt_wrapper(numba_func, return_type, dtype, dtype2=None, *, include_indexes=False): ztype = INT8 if return_type == BOOL else return_type xtype = INT8 if dtype == BOOL else dtype nt = numba.types + wrapper_args = [nt.CPointer(ztype.numba_type), nt.CPointer(xtype.numba_type)] + if include_indexes: + wrapper_args.extend([UINT64.numba_type, UINT64.numba_type]) if dtype2 is not None: ytype = INT8 if dtype2 == BOOL else dtype2 - wrapper_sig = nt.void( - nt.CPointer(ztype.numba_type), - nt.CPointer(xtype.numba_type), - nt.CPointer(ytype.numba_type), - ) - else: - wrapper_sig = nt.void( - nt.CPointer(ztype.numba_type), - nt.CPointer(xtype.numba_type), - ) + wrapper_args.append(nt.CPointer(ytype.numba_type)) + wrapper_sig = nt.void(*wrapper_args) - zarray = xarray = yarray = BL = BR = yarg = yname = "" + zarray = xarray = yarray = BL = BR = yarg = yname = rcidx = "" if return_type._is_udt: if return_type.np_type.subdtype is None: zarray = " z = numba.carray(z_ptr, 1)\n" @@ -1732,11 +1729,14 @@ def _get_udt_wrapper(numba_func, return_type, dtype, dtype2=None): else: yname = ", y_ptr[0]" + if include_indexes: + rcidx = ", row, col" + d = {"numba": numba, "numba_func": numba_func} text = ( - f"def wrapper(z_ptr, x_ptr{yarg}):\n" + f"def wrapper(z_ptr, x_ptr{rcidx}{yarg}):\n" f"{zarray}{xarray}{yarray}" - f" {zname} = {BL}numba_func({xname}{yname}){BR}\n" + f" {zname} = {BL}numba_func({xname}{rcidx}{yname}){BR}\n" ) exec(text, d) return d["wrapper"], wrapper_sig diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index 16166f3fb..cfb420725 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -1160,7 +1160,7 @@ def threex_minusthunk(x, row, col, thunk): def iii(x, row, col, thunk): return (row + col) // 2 >= thunk - indexunary.register_new("iii", iii) + select.register_new("iii", iii) assert hasattr(indexunary, "iii") assert hasattr(select, "iii") iii_apply = indexunary.register_anonymous(iii) diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index 512c47acb..ef01bb839 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -31,7 +31,7 @@ UINT64, ) from graphblas.exceptions import DomainMismatch, UdfParseError -from graphblas.operator import BinaryOp, Monoid, Semiring, UnaryOp, get_semiring +from graphblas.operator import BinaryOp, IndexUnaryOp, Monoid, Semiring, UnaryOp, get_semiring if dtypes._supports_complex: from graphblas.dtypes import FC32, FC64 @@ -1122,6 +1122,17 @@ def _udt_getx(val): expected = Vector.from_values([0, 1, 2], 0) assert result.isequal(expected) + def _udt_index(val, idx, _, thunk): + if idx == 0: # pragma: no cover + return thunk["y"] + return -thunk["y"] # pragma: no cover + + _udt_index = IndexUnaryOp.register_anonymous(_udt_index, "_udt_index", is_udt=True) + assert udt in _udt_index + result = v.apply(_udt_index, 3).new() + expected = Vector.from_values([0, 1, 2], [3, -3, -3]) + assert result.isequal(expected) + def _udt_first(x, y): return x # pragma: no cover diff --git a/graphblas/tests/test_pickle.py b/graphblas/tests/test_pickle.py index 476a902bc..d0d0227d5 100644 --- a/graphblas/tests/test_pickle.py +++ b/graphblas/tests/test_pickle.py @@ -23,6 +23,10 @@ def binaryanon(x, y): return x + y # pragma: no cover +def indexunaryanon(x, row, col, thunk): + return x >= thunk # pragma: no cover + + @pytest.mark.slow def test_deserialize(): thisdir = os.path.dirname(__file__) @@ -62,6 +66,8 @@ def test_serialize(): unary_anon = gb.operator.UnaryOp.register_anonymous(unaryanon) binary_anon = gb.operator.BinaryOp.register_anonymous(binaryanon) + indexunary_anon = gb.operator.IndexUnaryOp.register_anonymous(indexunaryanon) + select_anon = gb.operator.SelectOp.register_anonymous(indexunaryanon) monoid_anon = gb.operator.Monoid.register_anonymous(binary_anon, 0) semiring_anon = gb.operator.Semiring.register_anonymous(monoid_anon, binary_anon) d = { @@ -102,6 +108,8 @@ def test_serialize(): # "semiring_pickle": semiring_pickle, "unary_anon": unary_anon, "unary_anon[float]": unary_anon[float], + "indexunary_anon": indexunary_anon, + "select_anon": select_anon, "binary_anon": binary_anon, "monoid_anon": monoid_anon, "semiring_anon": semiring_anon, diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 69229647d..fadcaee84 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -703,7 +703,7 @@ def twox_minusthunk(x, row, col, thunk): def ii(x, idx, _, thunk): return idx // 2 >= thunk - indexunary.register_new("ii", ii) + select.register_new("ii", ii) assert hasattr(indexunary, "ii") assert hasattr(select, "ii") ii_apply = indexunary.register_anonymous(ii)