From 8c96f1908025588dac28aa4ada005865c1d6df87 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 25 Aug 2026 12:51:31 -0500 Subject: [PATCH 1/4] Adapt to SuiteSparse:GraphBLAS 10.5.0 and read its version from the library 10.5.0 stopped checking, after a build, whether every value happened to be the same: "GrB_Matrix_build and GrB_Vector_build: no longer do a post-iso check; they leave the matrix in non-iso format even if all the entries are the same." Four tests read that check as if it were python-graphblas's own contract and went red. Each one wanted an iso object rather than an inference, so each now asks for one with ``ss.build_scalar``, which means the same thing on every SuiteSparse version and produces the same storage formats the tests were selecting for. ``from_coo`` is left alone: adding a scan of the values to recover the old behavior would re-add the cost upstream just removed. The version gates had a second, quieter problem. ``graphblas.core.ss`` derived the C library version by parsing ``suitesparse_graphblas.__version__``, which is the version of the Python wrapper. The two normally agree, but a development build between releases reports the previous one: psg 10.4.1.0 against a 10.5.0 library here, so every gate saw 10.4.1 and any 10.5 branch would have been silently skipped. Read the library's own GxB_IMPLEMENTATION_* constants instead, falling back to the old parse where a build does not expose them. test_openmp_enabled keeps its hard assertion and gains a message saying what is wrong and that the fault is in the installed build. It caught a real one: conda-forge's first graphblas 10.5.0 build on osx-arm64 shipped without OpenMP. The build 1 rebuild (2026-08-26) restored OpenMP and the assertion passes again. --- graphblas/core/ss/__init__.py | 14 +++++++- graphblas/tests/test_formatting.py | 7 +++- graphblas/tests/test_io.py | 14 ++++++-- graphblas/tests/test_ss_utils.py | 58 +++++++++++++++++++++++++----- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/graphblas/core/ss/__init__.py b/graphblas/core/ss/__init__.py index 79859eb53..7ee1da469 100644 --- a/graphblas/core/ss/__init__.py +++ b/graphblas/core/ss/__init__.py @@ -1,5 +1,17 @@ import suitesparse_graphblas as _ssgb -version_major, version_minor, version_bug = map(int, _ssgb.__version__.split(".")[:3]) +# Version of the SuiteSparse:GraphBLAS C library, which is what every version +# gate in this package is really asking about. Read from the library's own +# constants rather than parsed out of ``_ssgb.__version__``: the wrapper's +# version normally tracks the library, but a development build between +# releases reports the previous one (10.4.1.0 against a 10.5.0 library), and a +# gate keyed on that takes the wrong branch without saying so. These are cffi +# ``#define`` constants, so they read at import time, before ``GrB_init``. +try: + version_major = _ssgb.lib.GxB_IMPLEMENTATION_MAJOR + version_minor = _ssgb.lib.GxB_IMPLEMENTATION_MINOR + version_bug = _ssgb.lib.GxB_IMPLEMENTATION_SUB +except AttributeError: # pragma: no cover (build without the GxB defines) + version_major, version_minor, version_bug = map(int, _ssgb.__version__.split(".")[:3]) _IS_SSGB7 = version_major == 7 diff --git a/graphblas/tests/test_formatting.py b/graphblas/tests/test_formatting.py index a6522dcef..2b412ac1d 100644 --- a/graphblas/tests/test_formatting.py +++ b/graphblas/tests/test_formatting.py @@ -5032,7 +5032,12 @@ def test_empty(): @pytest.mark.skipif("not pd") def test_vector_as_matrix(): - v = Vector.from_coo([1], [2], name="v_A") + # Built iso outright: the repr asserted below names the format, and + # SuiteSparse 10.5.0 no longer infers iso from a build whose values happen + # to be equal, so ``from_coo`` would render "bitmapc" on 10.5.0 and + # "bitmapc (iso)" on everything older. + v = Vector(int, 2, name="v_A") + v.ss.build_scalar([1], 2) A = v._as_matrix() repr_printer(A, "A") assert repr(A) == ( diff --git a/graphblas/tests/test_io.py b/graphblas/tests/test_io.py index a8d80e9ca..962595aaa 100644 --- a/graphblas/tests/test_io.py +++ b/graphblas/tests/test_io.py @@ -430,18 +430,28 @@ def test_awkward_roundtrip(): @pytest.mark.skipif("not ak") @pytest.mark.xfail(np.__version__[:5] in {"1.25.", "1.26."}, reason="awkward bug with numpy >=1.25") def test_awkward_iso_roundtrip(): + # Build iso outright rather than relying on all-equal values: SuiteSparse + # 10.5.0 no longer checks after a build whether every value is the same, + # so ``from_coo`` alone would leave these non-iso and the test would stop + # exercising the iso path it is named for. # Vector - v = gb.Vector.from_coo([1, 3, 5], [20, 20, 20], size=22) if suitesparse: + v = gb.Vector(int, 22) + v.ss.build_scalar([1, 3, 5], 20) assert v.ss.is_iso + else: + v = gb.Vector.from_coo([1, 3, 5], [20, 20, 20], size=22) kv = gb.io.to_awkward(v) assert isinstance(kv, ak.Array) v2 = gb.io.from_awkward(kv) assert v2.isequal(v) # Matrix - m = gb.Matrix.from_coo([0, 0, 3, 5], [1, 4, 0, 2], [1, 1, 1, 1], nrows=7, ncols=6) if suitesparse: + m = gb.Matrix(int, 7, 6) + m.ss.build_scalar([0, 0, 3, 5], [1, 4, 0, 2], 1) assert m.ss.is_iso + else: + m = gb.Matrix.from_coo([0, 0, 3, 5], [1, 4, 0, 2], [1, 1, 1, 1], nrows=7, ncols=6) for format in ["csr", "csc", "hypercsr", "hypercsc"]: km = gb.io.to_awkward(m, format=format) assert isinstance(km, ak.Array) diff --git a/graphblas/tests/test_ss_utils.py b/graphblas/tests/test_ss_utils.py index 40774186c..2c9e01a1a 100644 --- a/graphblas/tests/test_ss_utils.py +++ b/graphblas/tests/test_ss_utils.py @@ -10,6 +10,24 @@ pytest.skip("gb.ss and A.ss only available with suitesparse backend", allow_module_level=True) +# ``from_coo`` with all-equal values used to come back iso, because SuiteSparse +# checked after a build whether every value was the same. 10.5.0 dropped that +# check ("GrB_Matrix_build and GrB_Vector_build: no longer do a post-iso +# check"), so the iso arm of these tests has to ask for iso outright. +# ``ss.build_scalar`` does, and produces the same storage formats on every +# version, which is what the tests below are really selecting for. +def _iso_vector(indices, value, size): + v = Vector(int, size) + v.ss.build_scalar(indices, value) + return v + + +def _iso_matrix(rows, cols, value, nrows, ncols): + A = Matrix(int, nrows, ncols) + A.ss.build_scalar(rows, cols, value) + return A + + @pytest.mark.parametrize("do_iso", [False, True]) def test_vector_head(do_iso): v0 = Vector(int, 5) @@ -19,9 +37,14 @@ def test_vector_head(do_iso): values1 = [10, 20, 30] values2 = [2, 4, 6] values3 = [1, 2, 3] - v1 = Vector.from_coo([0, 1, 2], values1) # full - v2 = Vector.from_coo([1, 3, 5], values2) # bitmap - v3 = Vector.from_coo([100, 200, 300], values3) # sparse + if do_iso: + v1 = _iso_vector([0, 1, 2], values1[0], 3) # full + v2 = _iso_vector([1, 3, 5], values2[0], 6) # bitmap + v3 = _iso_vector([100, 200, 300], values3[0], 301) # sparse + else: + v1 = Vector.from_coo([0, 1, 2], values1) # full + v2 = Vector.from_coo([1, 3, 5], values2) # bitmap + v3 = Vector.from_coo([100, 200, 300], values3) # sparse assert v1.ss.export()["format"] == "full" assert v2.ss.export()["format"] == "bitmap" assert v3.ss.export()["format"] == "sparse" @@ -72,10 +95,16 @@ def test_matrix_head(do_iso): values2 = [1, 2, 4] values3 = values4 = [1, 2, 3] - A1 = Matrix.from_coo([0, 0, 1, 1], [0, 1, 0, 1], values1) # fullr - A2 = Matrix.from_coo([0, 0, 1], [0, 1, 1], values2) # Bitmap - A3 = Matrix.from_coo([5, 5, 10], [4, 5, 10], values3) # CSR - A4 = Matrix.from_coo([500, 500, 1000], [400, 500, 1000], values4) # HyperCSR + if do_iso: + A1 = _iso_matrix([0, 0, 1, 1], [0, 1, 0, 1], values1[0], 2, 2) # fullr + A2 = _iso_matrix([0, 0, 1], [0, 1, 1], values2[0], 2, 2) # Bitmap + A3 = _iso_matrix([5, 5, 10], [4, 5, 10], values3[0], 11, 11) # CSR + A4 = _iso_matrix([500, 500, 1000], [400, 500, 1000], values4[0], 1001, 1001) # HyperCSR + else: + A1 = Matrix.from_coo([0, 0, 1, 1], [0, 1, 0, 1], values1) # fullr + A2 = Matrix.from_coo([0, 0, 1], [0, 1, 1], values2) # Bitmap + A3 = Matrix.from_coo([5, 5, 10], [4, 5, 10], values3) # CSR + A4 = Matrix.from_coo([500, 500, 1000], [400, 500, 1000], values4) # HyperCSR d = A1.ss.export(raw=True) assert d["format"] == "fullr" d["format"] = "fullc" @@ -200,8 +229,19 @@ def test_about(): def test_openmp_enabled(): - # SuiteSparse:GraphBLAS without OpenMP enabled is very undesirable - assert gb.ss.about["openmp"] + # A SuiteSparse:GraphBLAS built without OpenMP is single-threaded, which is + # a large and entirely silent performance loss: nothing else about the + # library reports it, and the conda package still depends on an OpenMP + # runtime whether or not the build actually found one. Keep this assertion + # hard; it is how the conda-forge osx-arm64 regression in graphblas 10.5.0 + # was noticed. + assert gb.ss.about["openmp"], ( + "libgraphblas was built without OpenMP and will run single-threaded. " + "This is a defect in the installed build, not in python-graphblas. " + "Reinstall or rebuild graphblas with OpenMP enabled " + "(SuiteSparse only warns when its CMake cannot find OpenMP, so a " + "serial library can ship looking normal)." + ) def test_global_config(): From 7d995adb2cf9c425d631231dbf68c23e0072f2e2 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 25 Aug 2026 12:57:49 -0500 Subject: [PATCH 2/4] Cover SuiteSparse:GraphBLAS 10.4 and 10.5 in the CI psg pool The pool stopped at psg 10.3.1.0, so CI installed nothing newer except on the draw that picks "latest", which does not pin and so never produces a reproducible combination. Three releases have landed since, and each carries a SuiteSparse:GraphBLAS this project has to work against: 10.4.0 moved a field in the matrix struct, and 10.5.0 dropped the post-build iso check that four tests used to rely on. psg 10.5.0.0 is on PyPI and, since 2026-08-26, on conda-forge, so the conda-forge, wheel and source draws all cover it. `scripts/check_versions.sh` tracks the newest version we test, so its psg floor moves with the pool. Only the psg pools change here. The other pools are revised further up this branch stack, so raising them belongs there rather than in a commit that would conflict with every rebase. --- scripts/check_versions.sh | 2 +- scripts/ci_pick_versions.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/check_versions.sh b/scripts/check_versions.sh index 3977bd8a0..6905d26d9 100755 --- a/scripts/check_versions.sh +++ b/scripts/check_versions.sh @@ -17,5 +17,5 @@ conda search 'sparse[channel=conda-forge]>=0.15' # fast_matrix_market is deprecated (no longer maintained; last supported Python is 3.12) conda search 'numba[channel=conda-forge]>=0.64' conda search 'pyyaml[channel=conda-forge]>=6.0' -conda search 'python-suitesparse-graphblas[channel=conda-forge]>=10.3.1' +conda search 'python-suitesparse-graphblas[channel=conda-forge]>=10.5.0' # conda search 'python[channel=conda-forge]>=3.11 *pypy*' diff --git a/scripts/ci_pick_versions.py b/scripts/ci_pick_versions.py index 738ef99a8..b6cd1ea24 100755 --- a/scripts/ci_pick_versions.py +++ b/scripts/ci_pick_versions.py @@ -114,8 +114,17 @@ } PSG_VERSIONS_NP2 = { - "no_py314": ["9.3.1.0", "9.4.5.0", "10.0.1.1", "10.1.1.0", "10.3.1.0"], - "py314_only": ["10.0.1.1", "10.1.1.0", "10.3.1.0"], + "no_py314": [ + "9.3.1.0", + "9.4.5.0", + "10.0.1.1", + "10.1.1.0", + "10.3.1.0", + "10.4.0.0", + "10.4.1.0", + "10.5.0.0", + ], + "py314_only": ["10.0.1.1", "10.1.1.0", "10.3.1.0", "10.4.0.0", "10.4.1.0", "10.5.0.0"], } From 7b24f658fae7aeeeb92b6a66a5d79d5f001fd910 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 16 Sep 2026 10:30:31 -0500 Subject: [PATCH 3/4] Improvements to scripts/ci_pick_versions.py (AI-maintained) --- scripts/ci_pick_versions.py | 153 +++++++++++++++++++++++++++--------- 1 file changed, 116 insertions(+), 37 deletions(-) diff --git a/scripts/ci_pick_versions.py b/scripts/ci_pick_versions.py index b6cd1ea24..e479e715c 100755 --- a/scripts/ci_pick_versions.py +++ b/scripts/ci_pick_versions.py @@ -23,6 +23,13 @@ # "" means "latest" (no pin). "NA" means "don't install". # # When updating versions here, also update scripts/check_versions.sh +# +# To add a Python version (3.15, say): add it to `choices` in the "RNG for Python +# version" step of test_and_build.yml, give it a row in every table keyed by Python +# below (`check_python_pools` fails the run if you miss one), and revisit the floors +# in `apply_constraints`. Those floors are written as `_ver(pyver) >= (3, N)`, so a +# new Python inherits the previous one's floors instead of escaping them -- enough to +# keep CI honest, but usually still too loose to install, so check each dep. # --------------------------------------------------------------------------- NUMPY_VERSIONS = { @@ -66,6 +73,9 @@ "2.x": ["0.62", "0.63", "0.64", ""], } +# Oldest numba that supports each Python. +NUMBA_MIN_PYTHON = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} + # Deps that only depend on Python version (not numpy) NETWORKX_VERSIONS = { "3.11": ["2.8", "3.0", "3.1", "3.2", "3.3", "3.4", "3.5", "3.6", ""], @@ -113,18 +123,32 @@ }, } -PSG_VERSIONS_NP2 = { - "no_py314": [ - "9.3.1.0", - "9.4.5.0", - "10.0.1.1", - "10.1.1.0", - "10.3.1.0", - "10.4.0.0", - "10.4.1.0", - "10.5.0.0", - ], - "py314_only": ["10.0.1.1", "10.1.1.0", "10.3.1.0", "10.4.0.0", "10.4.1.0", "10.5.0.0"], +# PSG versions to pair with numpy 2.x, oldest first. Every source type except +# "upstream" draws from this one pool; Pythons too new for the older releases draw +# from a tail of it (see PSG_MIN_PYTHON). +PSG_VERSIONS_NP2 = [ + "9.3.1.0", + "9.4.5.0", + "10.0.1.1", + "10.1.1.0", + "10.3.1.0", + "10.4.0.0", + "10.4.1.0", + "10.5.0.0", +] + +# Oldest psg release that installs on a given Python, by where psg comes from. +# Pythons older than every entry here (3.11-3.13) can use the whole pool; a Python +# newer than every entry inherits the newest one until it gets a row of its own. +# +# py3.14 needs a different floor per source: conda-forge has py314 builds from +# 10.0.1.1 on, but PyPI's first cp314 wheels are in 10.3.1.0, so the "wheel" path +# can't reach back as far. "source" builds the sdist, and the sdists before 10.3.1.0 +# predate psg's py3.14 support (they still declare requires-python >=3.9), so hold +# them to the wheel floor rather than test a build psg never claimed works. +# MAINT: 2026-09-11 checked conda-forge build strings and PyPI wheel tags for psg 9-10 +PSG_MIN_PYTHON = { + "3.14": {"conda-forge": "10.0.1.1", "wheel": "10.3.1.0", "source": "10.3.1.0"}, } @@ -140,6 +164,19 @@ def _ver(s): return tuple(int(x) for x in s.split(".")) +def _min_for_python(table, pyver): + """Look up `pyver` in a {python version: minimum} table, or None if it predates it. + + A Python newer than every row inherits the newest row: it needs at least as new a + build as the newest Python we have characterized. That guess is conservative rather + than correct, so give a new Python its own row once its real floors are known. + """ + if pyver in table: + return table[pyver] + newest = max(table, key=_ver) + return table[newest] if _ver(pyver) > _ver(newest) else None + + # --------------------------------------------------------------------------- # Constraints # --------------------------------------------------------------------------- @@ -182,15 +219,15 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- scipy / Python version availability --- # scipy <1.14 has no py3.13 builds; scipy <1.16 has no py3.14 builds - if pyver == "3.14" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): + if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): v["scipy"] = random.choice(["1.16", "1.17", ""]) - elif pyver == "3.13" and v["scipy"] == "1.13": + elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": v["scipy"] = random.choice(["1.14", "1.15", "1.16", "1.17", ""]) # --- pandas constraints --- # pandas <2.3 has no py3.14 builds - if pyver == "3.14" and v["pandas"] == "2.2": + if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": v["pandas"] = random.choice(["2.3", "3.0", ""]) # pandas 3.0 requires numba >=0.60 and scipy >=1.14.1 @@ -227,9 +264,9 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- awkward / Python version availability --- # awkward <2.7 has no py3.13 builds; awkward <2.8 has no py3.14 builds - if pyver == "3.14" and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): + if _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): v["awkward"] = random.choice(["2.8", "2.9", ""]) - elif pyver == "3.13" and v["awkward"] == "2.6": + elif _ver(pyver) >= (3, 13) and v["awkward"] == "2.6": v["awkward"] = random.choice(["2.7", "2.8", "2.9", ""]) # --- awkward / numpy 2.5 deprecations --- @@ -276,10 +313,9 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- numba constraints --- - # numba minimum by Python version: 0.59 for 3.12, 0.61 for 3.13, 0.63 for 3.14 - numba_min = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} + # numba minimum by Python version (see NUMBA_MIN_PYTHON) if v["numba"] not in ("", "NA"): - min_ver = numba_min[pyver] + min_ver = _min_for_python(NUMBA_MIN_PYTHON, pyver) if _ver(v["numba"]) < min_ver: pool = [n for n in numba_pool if _ver(n) >= min_ver] v["numba"] = random.choice(pool) if pool else "" @@ -302,7 +338,7 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- sparse --- # sparse doesn't support Python 3.13+ - if pyver in ("3.13", "3.14"): + if _ver(pyver) >= (3, 13): v["sparse"] = "NA" @@ -393,7 +429,11 @@ def _pick_psg(npver, pyver, source_type): if not pool: return "" return f"{eq}{random.choice(pool)}" - pool = PSG_VERSIONS_NP2["py314_only"] if pyver == "3.14" else PSG_VERSIONS_NP2["no_py314"] + pool = PSG_VERSIONS_NP2 + floors = _min_for_python(PSG_MIN_PYTHON, pyver) + if floors is not None: + floor = _ver(floors[source_type]) + pool = [ver for ver in pool if _ver(ver) >= floor] return random.choice([f"{eq}{ver}" for ver in pool] + [""]) @@ -482,7 +522,31 @@ def format_summary(v): # --------------------------------------------------------------------------- -def validate(v, pyver): +def check_python_pools(): + """Check that every per-Python table covers the same Pythons. Returns list of errors. + + Adding a Python to some tables but not others would otherwise surface as a KeyError + from somewhere deep in the picking, or (for the tables consulted only by a + constraint) as a silently wrong pick. + """ + tables = { + "NETWORKX_VERSIONS": NETWORKX_VERSIONS, + "PYYAML_VERSIONS": PYYAML_VERSIONS, + "SPARSE_VERSIONS": SPARSE_VERSIONS, + "NUMBA_MIN_PYTHON": NUMBA_MIN_PYTHON, + } + errors = [] + for name, table in tables.items(): + missing = sorted(NUMPY_VERSIONS.keys() - table.keys(), key=_ver) + extra = sorted(table.keys() - NUMPY_VERSIONS.keys(), key=_ver) + if missing: + errors.append(f"{name} has no entry for Python {', '.join(missing)}") + if extra: + errors.append(f"{name} has entries for unsupported Python {', '.join(extra)}") + return errors + + +def validate(v, pyver, source_type=None): """Check that a version combination satisfies all constraints. Returns list of errors.""" errors = [] np_is_1x = v["numpy"].startswith("1.") if v["numpy"] else False @@ -504,14 +568,14 @@ def validate(v, pyver): errors.append("numpy 1.26 + scipy 1.9 conflict") # scipy Python availability - if pyver == "3.14" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): - errors.append(f"scipy {v['scipy']} has no py3.14 build") - if pyver == "3.13" and v["scipy"] == "1.13": - errors.append("scipy 1.13 has no py3.13 build") + if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): + errors.append(f"scipy {v['scipy']} has no py{pyver} build") + elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": + errors.append(f"scipy 1.13 has no py{pyver} build") # pandas Python availability - if pyver == "3.14" and v["pandas"] == "2.2": - errors.append("pandas 2.2 has no py3.14 build") + if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": + errors.append(f"pandas 2.2 has no py{pyver} build") # pandas 3.0 requirements if v["pandas"] == "3.0": @@ -530,10 +594,10 @@ def validate(v, pyver): errors.append(f"pandas {v['pandas']} constrains scipy >=1.10.0, got {v['scipy']}") # awkward Python availability - if pyver == "3.14" and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): - errors.append(f"awkward {v['awkward']} has no py3.14 build") - if pyver == "3.13" and v["awkward"] == "2.6": - errors.append("awkward 2.6 has no py3.13 build") + if _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): + errors.append(f"awkward {v['awkward']} has no py{pyver} build") + elif _ver(pyver) >= (3, 13) and v["awkward"] == "2.6": + errors.append(f"awkward 2.6 has no py{pyver} build") # awkward <2.6 requires numpy 1.x (numpy.AxisError) if not np_is_1x and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 6): @@ -567,8 +631,8 @@ def validate(v, pyver): errors.append(f"networkx {nx} requires pandas >=2.0, got {v['pandas']}") # numba Python minimums - numba_min = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} - if v["numba"] not in ("", "NA") and _ver(v["numba"]) < numba_min[pyver]: + numba_min = _min_for_python(NUMBA_MIN_PYTHON, pyver) + if v["numba"] not in ("", "NA") and _ver(v["numba"]) < numba_min: errors.append(f"numba {v['numba']} doesn't support Python {pyver}") # numba <0.64 requires numpy <2.4 @@ -580,9 +644,17 @@ def validate(v, pyver): errors.append(f"numba {v['numba']} doesn't support numpy 2.x") # sparse Python availability - if pyver in ("3.13", "3.14") and v["sparse"] != "NA": + if _ver(pyver) >= (3, 13) and v["sparse"] != "NA": errors.append(f"sparse doesn't support Python {pyver}") + # psg build availability (the floor depends on the Python and where psg comes from) + floors = _min_for_python(PSG_MIN_PYTHON, pyver) + if floors is not None and source_type in floors and v.get("psg"): + psgver = v["psg"].lstrip("=") + floor = floors[source_type] + if _ver(psgver) < _ver(floor): + errors.append(f"psg {psgver} has no py{pyver} {source_type} build (needs >={floor})") + return errors @@ -594,7 +666,7 @@ def stress_test(n=10000): for source in ("conda-forge", "wheel", "source", "upstream"): for _ in range(n): v = pick_versions(pyver, source) - errs = validate(v, pyver) + errs = validate(v, pyver, source) total += 1 if errs: failures += 1 @@ -627,6 +699,13 @@ def main(): if args.seed is not None: random.seed(args.seed) + pool_errors = check_python_pools() + if pool_errors: + print("Error: version pools disagree about which Pythons we support:", file=sys.stderr) + for e in pool_errors: + print(f" - {e}", file=sys.stderr) + sys.exit(1) + if args.validate: failures = stress_test() sys.exit(1 if failures else 0) From 5c98da46cf5e8ab9629e8df971c47ecc2fed716a Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 17 Sep 2026 16:00:59 -0500 Subject: [PATCH 4/4] Audit CI dependency pools against PyPI and conda-forge Add scripts/audit_{pypi,conda}_versions.py, which check the pools and tables in ci_pick_versions.py against what is actually published. The --validate stress test proves the picker agrees with the constraints it documents; it cannot notice when a constraint drifts from reality, since that happens when a package releases, not when we commit. A new weekly workflow runs both audits. In the picker, replace the hand-written scipy/numba compatibility rules with SCIPY_MAX_NUMPY and NUMBA_MAX_NUMPY ceiling tables, add SCIPY_MIN_PYTHON, and gate the tables behind check_ceilings() and check_python_pools() so a half-updated table fails the run instead of producing a silently wrong pick. Bump the pools to numpy 2.5, scipy 1.18, awkward 2.13, numba 0.67 and sparse 0.19, and offer sparse on py3.13/3.14 now that numba supports them. Also bump pre-commit hooks, update check_versions.sh to match, and ignore PLR0917, CPY and ISC004 (all newly stable in ruff 0.16). Fixes found while reviewing the above: - test_and_build.yml discarded the picker's exit status. `eval "$(cmd)"` reports eval's status, and that step's shell has no -e, so a picker failure left every *ver variable unset and quietly installed the latest of everything. Assign first, then fail the step. - pyproject: bound the sparse py3.14 marker below 3.15. sparse depends on numba, so it has to drop out wherever numba does. - _pick_scipy filtered ceilings and requires-python but not the one numpy floor scipy imposes, leaving the order dependence it exists to remove. - audit_pypi: report a missing scipy release instead of raising KeyError, honor a `<=` numpy cap rather than skipping the check, and keep the notes section inside the network guard. --- .github/workflows/audit_versions.yml | 80 +++++ .github/workflows/test_and_build.yml | 10 +- .pre-commit-config.yaml | 38 ++- pyproject.toml | 12 +- scripts/audit_conda_versions.py | 394 +++++++++++++++++++++++++ scripts/audit_pypi_versions.py | 424 +++++++++++++++++++++++++++ scripts/check_versions.sh | 19 +- scripts/ci_pick_versions.py | 306 ++++++++++++------- 8 files changed, 1154 insertions(+), 129 deletions(-) create mode 100644 .github/workflows/audit_versions.yml create mode 100755 scripts/audit_conda_versions.py create mode 100755 scripts/audit_pypi_versions.py diff --git a/.github/workflows/audit_versions.yml b/.github/workflows/audit_versions.yml new file mode 100644 index 000000000..646e96f59 --- /dev/null +++ b/.github/workflows/audit_versions.yml @@ -0,0 +1,80 @@ +# Checks the CI dependency pools in scripts/ci_pick_versions.py against the world. +# +# `ci_pick_versions.py --validate` runs in pre-commit and proves the picker agrees with +# the constraints it documents. It cannot notice when a constraint has drifted from +# reality, because that happens when a package releases, not when we commit. So this +# workflow runs weekly, and on changes to the picker or the audits themselves. +# +# A failure here does not mean CI is broken right now. It means a pool or a table no +# longer matches what PyPI or conda-forge publishes, and the next random pick that +# lands on the stale combination will fail for reasons that look unrelated. + +name: Audit dependency versions + +on: + schedule: + # Mondays at 06:00 UTC, before most of the week's PRs + - cron: "0 6 * * 1" + workflow_dispatch: + pull_request: + paths: + - scripts/ci_pick_versions.py + - scripts/audit_pypi_versions.py + - scripts/audit_conda_versions.py + - .github/workflows/audit_versions.yml + push: + branches: + - main + paths: + - scripts/ci_pick_versions.py + - scripts/audit_pypi_versions.py + - scripts/audit_conda_versions.py + - .github/workflows/audit_versions.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + pypi: + name: Audit against PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip packaging + - name: Stress-test the picker + run: python scripts/ci_pick_versions.py --validate + - name: Audit pools against PyPI + run: python scripts/audit_pypi_versions.py --samples 200 + + conda: + name: Audit against conda-forge + runs-on: ubuntu-latest + # conda-forge is what CI installs from, so this is the one that binds. It needs a + # conda to search with, which is why it is a separate, slower job. + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: "3.12" + channels: conda-forge + conda-remove-defaults: true + channel-priority: strict + activate-environment: audit + - run: python -m pip install --upgrade pip packaging + - name: Audit pools against conda-forge + run: python scripts/audit_conda_versions.py --samples 100 diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 69fab3770..574bcd3d8 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -145,7 +145,15 @@ jobs: # Pick random compatible dependency versions using the version picker script. # See scripts/ci_pick_versions.py for version pools and constraint logic. pyver=$(python -c "v='${{ steps.pyver.outputs.selected }}'.split('.');print(f'{v[0]}.{v[1]}')") - eval "$(python scripts/ci_pick_versions.py --python "$pyver" --source ${{ steps.sourcetype.outputs.selected }})" + # Assign first, then eval. `eval "$(cmd)"` reports eval's status, not cmd's, and + # this step's shell has no `-e`, so a picker that exits non-zero (a pool table it + # rejects, say) would leave every *ver variable unset and quietly install the + # latest of everything: a green run that tested nothing it claims to have tested. + if ! picks=$(python scripts/ci_pick_versions.py --python "$pyver" --source ${{ steps.sourcetype.outputs.selected }}) ; then + echo "::error::ci_pick_versions.py failed; refusing to install unpinned dependencies" + exit 1 + fi + eval "$picks" # Set up psg conda package variable (conda-forge installs via conda, others via pip). # For conda-forge, psg must always be installed; an empty psgver just means "latest". diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3582142d2..3b36390ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,23 +42,20 @@ repos: - id: name-tests-test args: ["--pytest-test-first"] - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.25 + rev: "0.26" hooks: - id: validate-pyproject name: Validate pyproject.toml # I don't yet trust ruff to do what autoflake does - repo: https://github.com/PyCQA/autoflake - rev: v2.3.3 + rev: v2.4.0 hooks: - id: autoflake args: [--in-place] # We can probably remove `isort` if we come to trust `ruff --fix`, # but we'll need to figure out the configuration to do this in `ruff` - repo: https://github.com/pycqa/isort - # 9.0 is still in alpha (9.0.0a3) and a pre-release pin defeats - # pre-commit.ci's autoupdate "avoid prereleases" logic; stay on the last - # stable 8.x until 9.0 GAs. - rev: 8.0.1 + rev: 9.0.1 hooks: - id: isort # Let's keep `pyupgrade` even though `ruff --fix` probably does most of it @@ -73,12 +70,12 @@ repos: - id: auto-walrus args: [--line-length, "100"] - repo: https://github.com/psf/black-pre-commit-mirror - rev: 26.3.1 + rev: 26.5.1 hooks: - id: black - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.13 + rev: v0.16.8 hooks: - id: ruff-check args: [--fix-only, --show-fixes] @@ -92,10 +89,10 @@ repos: additional_dependencies: &flake8_dependencies # These versions need updated manually - flake8==7.3.0 - - flake8-bugbear==25.11.29 + - flake8-bugbear==26.9.9 - flake8-simplify==0.30.0 - repo: https://github.com/codespell-project/codespell - rev: v2.4.2 + rev: v2.4.3 hooks: - id: codespell types_or: [python, markdown, rst, toml, yaml] @@ -103,7 +100,7 @@ repos: - tomli; python_version<'3.11' files: ^(graphblas|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.13 + rev: v0.16.8 hooks: - id: ruff-check # - id: ruff-format # we prefer black; uncomment to compare with ruff's formatter @@ -115,16 +112,18 @@ repos: # `pyroma` may help keep our package standards up to date if best practices change. # This is probably a "low value" check though and safe to remove if we want faster pre-commit. - repo: https://github.com/regebro/pyroma + # 5.1 is still in beta (5.1b2) and a pre-release pin defeats pre-commit.ci's + # autoupdate "avoid prereleases" logic; stay on 5.0.1 until 5.1 GAs. rev: "5.0.1" hooks: - id: pyroma args: [-n, "10", .] - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.11.0.1" + rev: "v0.11.0.1-1" hooks: - id: shellcheck - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.8.3 + rev: v3.9.7 hooks: - id: prettier args: [--prose-wrap=preserve] @@ -149,7 +148,7 @@ repos: hooks: - id: actionlint - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.37.2 + rev: 0.38.0 hooks: - id: check-dependabot - id: check-github-workflows @@ -163,7 +162,7 @@ repos: # hooks: # - id: zizmor - repo: https://github.com/pysentry/pysentry-pre-commit - rev: v0.4.6 + rev: v0.5.0 hooks: - id: pysentry args: ["--format=human", "--fail-on=low", "--sources=pypa,osv,pypi"] @@ -206,6 +205,15 @@ repos: | scripts/find_aiisms\.sh | \.pre-commit-config\.yaml )$ + # Stress-tests every combination the CI dependency picker can produce against + # the constraints it documents. ~3s for 160k combinations, and it only runs + # when the picker itself changes. + - id: ci-pick-versions + name: Validate CI dependency version picker + entry: python scripts/ci_pick_versions.py --validate + language: system + files: ^scripts/ci_pick_versions\.py$ + pass_filenames: false # Add `--hook-stage manual` to pre-commit command to run (very slow) # It's probably better (and faster!) to simply run `pylint graphblas/some/file.py` - id: pylint diff --git a/pyproject.toml b/pyproject.toml index 9fe272c98..cdf8867c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,14 @@ io = [ "python-graphblas[networkx,scipy]", "python-graphblas[numba]; python_version<'3.15'", # make optional where numba is not supported "awkward >=2.0", - "sparse >=0.14; python_version<'3.13'", # make optional, b/c sparse needs numba + # sparse needs numba, so it stays optional. numba gained py3.13 support in 0.61 and + # py3.14 in 0.63; sparse 0.16 is the first release that postdates py3.13, and 0.18 the + # first that postdates py3.14. + "sparse >=0.14; python_version<'3.13'", + "sparse >=0.16; python_version=='3.13'", + # Upper guard for the same reason as numba above: sparse depends on numba, so it has + # to drop out wherever numba does, not just trail it. + "sparse >=0.18; python_version>='3.14' and python_version<'3.15'", # fast-matrix-market is no longer maintained; last supported Python is 3.12 ] viz = ["python-graphblas[networkx,scipy]", "matplotlib >=3.6"] @@ -342,6 +349,7 @@ ignore = [ "PLR0911", # Too many return statements "PLR0912", # Too many branches "PLR0913", # Too many arguments to function call + "PLR0917", # Too many positional arguments (Stable since ruff 0.16; same call for PLR0913) "PLR0915", # Too many statements "PLR2004", # Magic number used in comparison, consider replacing magic with a constant variable "PLW0603", # Using the global statement to update ... is discouraged (Note: yeah, discouraged, but too strict) @@ -362,6 +370,7 @@ ignore = [ # Ignored categories "C90", # mccabe (Too strict, but maybe we should make things less complex) + "CPY", # flake8-copyright (Stable since ruff 0.16; we don't put a header in every file) "I", # isort (Should we replace `isort` with this?) "ANN", # flake8-annotations (We don't use annotations yet) "BLE", # flake8-blind-except (Maybe consider) @@ -394,6 +403,7 @@ ignore = [ "NPY002", # numba doesn't support rng generator yet ] "graphblas/core/utils.py" = ["PLE0302"] # `__set__` is used as a property +"graphblas/core/formatting.py" = ["ISC004"] # Adjacent literals form one HTML fragment "scripts/ci_pick_versions.py" = ["S311"] # random.choice is intentional for CI version selection "graphblas/ss/_core.py" = ["N999"] # We want _core.py to be underscopre # Allow useless expressions, assert, pickle, RNG, print, no docstring, and yoda in tests diff --git a/scripts/audit_conda_versions.py b/scripts/audit_conda_versions.py new file mode 100755 index 000000000..0d2fe4c16 --- /dev/null +++ b/scripts/audit_conda_versions.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python +"""Audit the CI dependency pools in `ci_pick_versions.py` against conda-forge. + +CI installs its dependencies from conda-forge, so this is the authoritative companion +to `audit_pypi_versions.py`. It can see two things PyPI cannot: + +* which Pythons a version was actually built for. conda-forge builds more than PyPI + ships (it has a py311 build of pyyaml 5.4.1 where PyPI's newest wheel is cp39), and + sometimes fewer, so "no wheel" and "no build" are different questions. +* `constrains` metadata (conda's run_constrained). Several rules in the picker exist + only because a package constrains something it does not depend on, which is what + makes a pinned pair fail to solve. + +Usage: + python scripts/audit_conda_versions.py [--samples N] + +Requires `conda` on PATH and network access. Slower than the PyPI audit: one +`conda search` per package, a few seconds each. Exits non-zero if any check fails. +See also scripts/check_versions.sh for the by-hand version of the same questions. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import random +import re +import shutil +import subprocess +import sys +from pathlib import Path + +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import InvalidVersion, Version + +_PICKER = Path(__file__).with_name("ci_pick_versions.py") +_spec = importlib.util.spec_from_file_location("ci_pick_versions", _PICKER) +picker = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(picker) + +# Our internal names to conda-forge package names +CONDA_NAMES = {"psg": "python-suitesparse-graphblas", "matplotlib": "matplotlib-base"} + +PER_PYTHON_POOLS = { + "numpy": picker.NUMPY_VERSIONS, + "networkx": picker.NETWORKX_VERSIONS, + "pyyaml": picker.PYYAML_VERSIONS, + "sparse": picker.SPARSE_VERSIONS, +} + +SAMPLED_DEPS = ("scipy", "pandas", "awkward", "numba", "networkx", "sparse", "pyyaml", "psg") + +# Constraints the picker hard-codes because they come from conda metadata alone. If one +# of these drifts, the matching rule in `apply_constraints` needs another look. +# Each entry is (dep, version pin, field, expected constraint). +DOCUMENTED_METADATA = [ + ("networkx", "3.6", "constrains", "numpy >=1.25"), + ("networkx", "3.6", "constrains", "scipy >=1.11.2"), + ("networkx", "3.6", "constrains", "pandas >=2.0"), + ("networkx", "3.5", "constrains", "numpy >=1.25"), + ("pandas", "2.3", "constrains", "scipy >=1.10.0"), + # matplotlib 3.11.0's metadata understates its numpy floor; 3.11.1 fixed it. The + # picker pins matplotlib <3.11 for old numpy because of the 3.11.0 builds. + ("matplotlib", "3.11.0", "depends", "numpy >=1.23"), + ("matplotlib", "3.11.1", "depends", "numpy >=1.25"), +] + + +# --------------------------------------------------------------------------- +# conda access +# --------------------------------------------------------------------------- + +_cache: dict[tuple[str, str], list] = {} + + +def conda_search(dep, floor): + """All conda-forge builds of `dep` at or above `floor`, cached per process.""" + if (dep, floor) not in _cache: + name = CONDA_NAMES.get(dep, dep) + spec = f"{name}[channel=conda-forge]>={floor}" + result = subprocess.run( + [_CONDA, "search", "--json", spec], + capture_output=True, + text=True, + check=False, + timeout=600, + ) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError: + raise SystemExit(f"conda search failed for {spec}:\n{result.stderr.strip()}") from None + if isinstance(payload, dict) and payload.get("error"): + raise SystemExit(f"conda search failed for {spec}: {payload['error']}") + _cache[dep, floor] = [build for builds in payload.values() for build in builds] + return _cache[dep, floor] + + +def builds_for(dep, pin, floor): + """Builds of `dep` matching a pin ("" means every build at or above `floor`).""" + return [ + b + for b in conda_search(dep, floor) + if not pin or b["version"] == pin or b["version"].startswith(f"{pin}.") + ] + + +def build_supports(build, pyver): + """Whether a conda build can be installed on `pyver`.""" + minor = int(pyver.split(".")[1]) + if match := re.search(r"py3(\d\d)", build["build"]): + return int(match.group(1)) == minor + constraints = [] + for dep in build.get("depends", []): + tokens = dep.split() + if tokens[0] != "python" or len(tokens) < 2: + continue + constraints.append(f"=={tokens[1]}" if tokens[1].endswith(".*") else tokens[1]) + if not constraints: + return True # noarch with a bare `python` dep + try: + return Version(pyver) in SpecifierSet(",".join(constraints)) + except (InvalidSpecifier, InvalidVersion): + return True + + +def supports_python(dep, pin, pyver, floor): + """Whether any build matching the pin supports `pyver`.""" + return any(build_supports(b, pyver) for b in builds_for(dep, pin, floor)) + + +def numpy_cap(dep, pin, floor): + """The tightest `numpy declared: + problems.append( + f"{dep} {pin} caps numpy at {declared} on conda-forge, " + f"but the table allows up to {ceiling}" + ) + return problems + + +def note_conservative_ceilings(): + """Ceiling rows stricter than conda-forge's cap. Advisory: coverage left on the table.""" + notes = [] + for dep, table in (("scipy", picker.SCIPY_MAX_NUMPY), ("numba", picker.NUMBA_MAX_NUMPY)): + floor = _oldest_pin(dep) + for pin, ceiling in table.items(): + declared = numpy_cap(dep, pin, floor) + if declared is not None and ceiling < declared: + notes.append( + f"{dep} {pin} is capped at {ceiling} in the table, " + f"but conda-forge allows numpy up to {declared}" + ) + return notes + + +def check_psg_python_floors(): + """PSG_MIN_PYTHON's conda-forge floor must be the oldest psg built for that Python.""" + problems = [] + floor = _oldest_pin("psg") + for pyver, sources in picker.PSG_MIN_PYTHON.items(): + expected = sources.get("conda-forge") + built = [p for p in picker.PSG_VERSIONS_NP2 if supports_python("psg", p, pyver, floor)] + if built and min(built, key=picker._ver) != expected: + problems.append( + f"PSG_MIN_PYTHON says conda-forge py{pyver} starts at psg {expected}, " + f"but {min(built, key=picker._ver)} has a py{pyver} build" + ) + return problems + + +def check_documented_metadata(): + """Constraints the picker's rules quote must still be what conda-forge publishes.""" + problems = [] + for dep, pin, field, expected in DOCUMENTED_METADATA: + matching = builds_for(dep, pin, _oldest_pin(dep)) + if not matching: + problems.append(f"{dep} {pin} has no conda-forge build; cannot confirm {expected!r}") + continue + if not any(expected in build.get(field, []) for build in matching): + problems.append(f"no {dep} {pin} build declares {field} {expected!r} any more") + return problems + + +def report_newer_releases(): + """conda-forge versions newer than our newest pin. Informational, never a failure.""" + news = [] + for dep in ( + "numpy", + "scipy", + "pandas", + "awkward", + "numba", + "networkx", + "pyyaml", + "sparse", + "psg", + ): + floor = _oldest_pin(dep) + newest_pin = max( + (b["version"] for b in builds_for(dep, "", floor)), key=_version_key, default=None + ) + tested = _newest_tested(dep) + if not newest_pin or not tested: + continue + # A pin names a series, so "2.5" already covers conda-forge's 2.5.3 + in_tested_series = newest_pin == tested or newest_pin.startswith(f"{tested}.") + if not in_tested_series and _version_key(newest_pin) > _version_key(tested): + news.append(f"{dep}: newest pin is {tested}, conda-forge has {newest_pin}") + return news + + +def _newest_tested(dep): + pools = { + "numpy": picker.NUMPY_VERSIONS, + "scipy": {"": picker.SCIPY_VERSIONS["2.x"]}, + "pandas": {"": picker.PANDAS_VERSIONS["2.x"]}, + "awkward": {"": picker.AWKWARD_VERSIONS["2.x"]}, + "numba": {"": picker.NUMBA_VERSIONS["2.x"]}, + "networkx": picker.NETWORKX_VERSIONS, + "pyyaml": picker.PYYAML_VERSIONS, + "sparse": picker.SPARSE_VERSIONS, + "psg": {"": picker.PSG_VERSIONS_NP2}, + }[dep] + pins = {p for group in pools.values() for p in group if p and p != "NA"} + return max(pins, key=picker._ver) if pins else None + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +_CONDA = shutil.which("conda") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--samples", + type=int, + default=100, + help="random picks per Python/source combination (default: 100)", + ) + parser.add_argument("--seed", type=int, default=None, help="seed for reproducible sampling") + args = parser.parse_args() + if _CONDA is None: + print("conda is not on PATH; this audit needs it", file=sys.stderr) + return 2 + if args.seed is not None: + random.seed(args.seed) + + checks = ( + ("per-Python pools", check_per_python_pools), + ("sampled picks", lambda: check_sampled_picks(args.samples)), + ("numpy ceiling tables", check_numpy_ceilings), + ("psg Python floors", check_psg_python_floors), + ("documented conda metadata", check_documented_metadata), + ) + failures = 0 + for name, check in checks: + problems = check() + if problems: + failures += len(problems) + print(f"FAIL {name}:") + for problem in problems: + print(f" - {problem}") + else: + print(f"ok {name}") + + if notes := note_conservative_ceilings(): + print("\nNotes (stricter than conda-forge needs; often right, since conda's depends") + print("carry ABI caps while a package's real runtime cap lives upstream):") + for note in notes: + print(f" - {note}") + + if news := report_newer_releases(): + print("\nNewer versions exist (not a failure; update the pools when convenient):") + for line in news: + print(f" - {line}") + + print(f"\n{failures} problem(s) found", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/audit_pypi_versions.py b/scripts/audit_pypi_versions.py new file mode 100755 index 000000000..75cc202e9 --- /dev/null +++ b/scripts/audit_pypi_versions.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python +"""Audit the CI dependency pools in `ci_pick_versions.py` against PyPI. + +`ci_pick_versions.py --validate` proves the picker agrees with the constraints it +documents. It cannot notice when a constraint has drifted from reality: a pin that +never had a build for a Python we test, or a ceiling table that a new release moved. +This script checks the pools and tables against what the packages actually declare. + +The picker's rules are written for conda-forge, which is what CI installs from, so +treat PyPI as a fast proxy and `audit_conda_versions.py` as the authority. Anything +this script reports is worth looking at in both places. + +Usage: + python scripts/audit_pypi_versions.py [--samples N] + +Requires network access and `packaging`. Exits non-zero if any check fails. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import random +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.version import InvalidVersion, Version + +# Import the picker as a module so the pools stay the single source of truth +_PICKER = Path(__file__).with_name("ci_pick_versions.py") +_spec = importlib.util.spec_from_file_location("ci_pick_versions", _PICKER) +picker = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(picker) + +# Our internal names to PyPI project names (conda-forge calls psg python-suitesparse-graphblas) +PYPI_NAMES = {"pyyaml": "PyYAML", "psg": "suitesparse-graphblas"} + +# Pools keyed by Python version, where every pin is offered on that Python and nowhere else +PER_PYTHON_POOLS = { + "numpy": picker.NUMPY_VERSIONS, + "networkx": picker.NETWORKX_VERSIONS, + "pyyaml": picker.PYYAML_VERSIONS, + "sparse": picker.SPARSE_VERSIONS, +} + +# Deps a pick can carry, in the order the picker fills them in +SAMPLED_DEPS = ("scipy", "pandas", "awkward", "numba", "networkx", "sparse", "pyyaml", "psg") + + +# --------------------------------------------------------------------------- +# PyPI access +# --------------------------------------------------------------------------- + +_cache: dict[str, dict] = {} + + +def _pypi(path): + """GET a PyPI JSON endpoint, cached for the life of the process.""" + if path not in _cache: + url = f"https://pypi.org/pypi/{path}/json" + request = urllib.request.Request(url, headers={"User-Agent": "python-graphblas-audit"}) + with urllib.request.urlopen(request, timeout=90) as response: # noqa: S310 + _cache[path] = json.load(response) + return _cache[path] + + +def releases(dep): + """Every non-prerelease, non-yanked release of `dep`: {Version: {...}}.""" + key = f"releases:{dep}" + if key not in _cache: + data = _pypi(PYPI_NAMES.get(dep, dep)) + out = {} + for raw, files in data["releases"].items(): + files = [f for f in files if not f["yanked"]] + if not files: + continue + try: + version = Version(raw) + except InvalidVersion: + continue + if version.is_prerelease: + continue + out[version] = { + "requires_python": files[0].get("requires_python"), + "cpython": {int(m) for f in files for m in re.findall(r"cp3(\d+)", f["filename"])}, + } + _cache[key] = out + return _cache[key] + + +def in_series(dep, pin): + """Every release matching a pin. An empty pin means "latest", so every release.""" + return [v for v in releases(dep) if not pin or str(v) == pin or str(v).startswith(f"{pin}.")] + + +def newest_in_series(dep, pin, pyver=None): + """The release a pin resolves to, preferring one that `pyver` can install. + + Returns (version, installable). When no release in the series supports `pyver` the + newest one is returned with installable=False, so callers can still read its + metadata while reporting the gap. + """ + found = in_series(dep, pin) + if not found: + return None, False + if pyver is not None: + usable = [v for v in found if supports_python(dep, v, pyver)] + if usable: + return max(usable), True + return max(found), False + return max(found), True + + +def supports_python(dep, version, pyver): + """Whether `version` of `dep` can be installed on `pyver`. + + Projects that ship compiled wheels are judged by their CPython tags, which is the + only signal that catches a release whose requires-python is wide but that simply + never built for this interpreter. Pure-Python projects are judged by requires-python. + """ + info = releases(dep)[version] + if info["cpython"]: + return int(pyver.split(".")[1]) in info["cpython"] + requires = info["requires_python"] + return not requires or Version(pyver) in SpecifierSet(requires) + + +def numpy_bound(dep, version, pyver): + """The numpy requirement `version` of `dep` declares for `pyver`, if any. + + A project may list numpy more than once (numba 0.65 declares both `numpy>=1.22` and + `numpy<2.5,>=1.22`), so combine every applicable clause rather than taking the first. + """ + data = _pypi(f"{PYPI_NAMES.get(dep, dep)}/{version}") + combined = None + for raw in data["info"].get("requires_dist") or []: + requirement = Requirement(raw) + if requirement.name.lower() != "numpy": + continue + marker = requirement.marker + if marker is None or marker.evaluate({"python_version": pyver, "extra": ""}): + combined = ( + requirement.specifier if combined is None else combined & requirement.specifier + ) + return combined + + +# --------------------------------------------------------------------------- +# Checks +# --------------------------------------------------------------------------- + + +def check_per_python_pools(): + """Every pin in a per-Python pool must exist on PyPI at all.""" + problems = [] + for dep, pools in PER_PYTHON_POOLS.items(): + for pyver, pins in pools.items(): + for pin in pins: + if pin == "NA": + continue + version, _ = newest_in_series(dep, pin) + if version is None: + problems.append(f"{dep} {pin} in the py{pyver} pool has no release on PyPI") + return problems + + +def note_pypi_python_gaps(): + """Pins with no PyPI build for a Python we offer them on. Advisory, not a failure. + + CI installs these from conda-forge, which builds more than PyPI ships: conda-forge + has a py311 build of pyyaml 5.4.1 where PyPI's newest wheel is cp39. Treat a note + here as a prompt to check `audit_conda_versions.py`, not as a broken pool. + """ + notes = [] + for dep, pools in PER_PYTHON_POOLS.items(): + for pyver, pins in pools.items(): + for pin in pins: + if pin == "NA": + continue + version, installable = newest_in_series(dep, pin, pyver) + if version is not None and not installable: + notes.append( + f"{dep} {pin or 'latest'} is offered on py{pyver}, " + f"but PyPI's {version} has no py{pyver} build" + ) + return notes + + +def check_ceiling_tables(): + """A ceiling row must never sit above the cap the release actually declares. + + A row above the real cap lets through a pairing that breaks; a row below it is + conservative, which is often deliberate, so that is reported as a note instead. + """ + problems = [] + tables = (("scipy", picker.SCIPY_MAX_NUMPY), ("numba", picker.NUMBA_MAX_NUMPY)) + for dep, table in tables: + for pin, ceiling in table.items(): + version, _ = newest_in_series(dep, pin) + if version is None: + problems.append(f"{dep} {pin} has a ceiling row but no release on PyPI") + continue + # Read the bound on the oldest Python we test, where markers are least likely + # to narrow it, then compare against the "< X.Y" the table records. + specifier = numpy_bound(dep, version, min(picker.NUMPY_VERSIONS, key=picker._ver)) + declared = _upper_bound(specifier) + if declared is not None and ceiling > declared: + problems.append( + f"{dep} {pin} ({version}) caps numpy at {declared}, " + f"but the table allows up to {ceiling}" + ) + return problems + + +def note_conservative_ceilings(): + """Ceiling rows stricter than PyPI's cap. Advisory: usually a conda-forge cap.""" + notes = [] + for dep, table in (("scipy", picker.SCIPY_MAX_NUMPY), ("numba", picker.NUMBA_MAX_NUMPY)): + for pin, ceiling in table.items(): + version, _ = newest_in_series(dep, pin) + if version is None: + continue + declared = _upper_bound( + numpy_bound(dep, version, min(picker.NUMPY_VERSIONS, key=picker._ver)) + ) + if declared is not None and ceiling < declared: + notes.append( + f"{dep} {pin} is capped at {ceiling} in the table, " + f"but PyPI's {version} allows numpy up to {declared}" + ) + return notes + + +def _upper_bound(specifier): + """The ` resolved and not str(latest).startswith(newest_pin): + news.append(f"{dep}: newest pin is {newest_pin} ({resolved}), PyPI has {latest}") + return news + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--samples", + type=int, + default=100, + help="random picks per Python/source combination (default: 100)", + ) + parser.add_argument("--seed", type=int, default=None, help="seed for reproducible sampling") + args = parser.parse_args() + if args.seed is not None: + random.seed(args.seed) + + checks = ( + ("per-Python pools", check_per_python_pools), + ("numpy ceiling tables", check_ceiling_tables), + ("Python floor tables", check_python_floor_tables), + ("sampled picks", lambda: check_sampled_picks(args.samples)), + ) + failures = 0 + for name, check in checks: + try: + problems = check() + except (urllib.error.URLError, TimeoutError) as exc: + print(f"SKIP {name}: PyPI unreachable ({exc})", file=sys.stderr) + continue + if problems: + failures += len(problems) + print(f"FAIL {name}:") + for problem in problems: + print(f" - {problem}") + else: + print(f"ok {name}") + + try: + notes = note_conservative_ceilings() + note_pypi_python_gaps() + news = report_newer_releases() + except (urllib.error.URLError, TimeoutError) as exc: + print(f"SKIP notes: PyPI unreachable ({exc})", file=sys.stderr) + notes = news = [] + + if notes: + print("\nNotes (PyPI only; conda-forge builds more, so check the conda audit):") + for note in notes: + print(f" - {note}") + + if news: + print("\nNewer releases exist (not a failure; update the pools when convenient):") + for line in news: + print(f" - {line}") + + print(f"\n{failures} problem(s) found", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_versions.sh b/scripts/check_versions.sh index 6905d26d9..d9959556d 100755 --- a/scripts/check_versions.sh +++ b/scripts/check_versions.sh @@ -6,16 +6,23 @@ # # When updating versions throughout the repo (CI, pyproject.toml, pre-commit, etc.), # also update these version numbers to match the latest versions we currently test. -conda search 'flake8-bugbear[channel=conda-forge]>=25.11.29' +# +# scripts/audit_pypi_versions.py and scripts/audit_conda_versions.py ask the same +# questions automatically, and also check the pools and tables in ci_pick_versions.py +# against what is published. The "Audit dependency versions" workflow runs both weekly. +conda search 'flake8-bugbear[channel=conda-forge]>=26.9.9' conda search 'flake8-simplify[channel=conda-forge]>=0.30.0' -conda search 'numpy[channel=conda-forge]>=2.4' +conda search 'numpy[channel=conda-forge]>=2.5' conda search 'pandas[channel=conda-forge]>=3.0' -conda search 'scipy[channel=conda-forge]>=1.17' +conda search 'scipy[channel=conda-forge]>=1.18' conda search 'networkx[channel=conda-forge]>=3.6' -conda search 'awkward[channel=conda-forge]>=2.11' -conda search 'sparse[channel=conda-forge]>=0.15' +conda search 'awkward[channel=conda-forge]>=2.13' +conda search 'sparse[channel=conda-forge]>=0.19' # fast_matrix_market is deprecated (no longer maintained; last supported Python is 3.12) -conda search 'numba[channel=conda-forge]>=0.64' +conda search 'numba[channel=conda-forge]>=0.67' conda search 'pyyaml[channel=conda-forge]>=6.0' +# matplotlib is installed unpinned for the notebooks job; ci_pick_versions.py caps it +# below 3.11 when the numpy pick predates matplotlib's numpy >=1.25 runtime floor. +conda search 'matplotlib-base[channel=conda-forge]>=3.11' conda search 'python-suitesparse-graphblas[channel=conda-forge]>=10.5.0' # conda search 'python[channel=conda-forge]>=3.11 *pypy*' diff --git a/scripts/ci_pick_versions.py b/scripts/ci_pick_versions.py index e479e715c..2fcfafc02 100755 --- a/scripts/ci_pick_versions.py +++ b/scripts/ci_pick_versions.py @@ -28,15 +28,16 @@ # version" step of test_and_build.yml, give it a row in every table keyed by Python # below (`check_python_pools` fails the run if you miss one), and revisit the floors # in `apply_constraints`. Those floors are written as `_ver(pyver) >= (3, N)`, so a -# new Python inherits the previous one's floors instead of escaping them -- enough to -# keep CI honest, but usually still too loose to install, so check each dep. +# new Python inherits the previous one's floors instead of escaping them. That is +# enough to keep CI honest, but usually still too loose to install, so check each dep. # --------------------------------------------------------------------------- +# numpy 2.5 requires Python >=3.12, so py3.11 stops at 2.4. NUMPY_VERSIONS = { "3.11": ["1.24", "1.25", "1.26", "2.0", "2.1", "2.2", "2.3", "2.4", ""], - "3.12": ["1.26", "2.0", "2.1", "2.2", "2.3", "2.4", ""], - "3.13": ["2.1", "2.2", "2.3", "2.4", ""], - "3.14": ["2.3", "2.4", ""], + "3.12": ["1.26", "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", ""], + "3.13": ["2.1", "2.2", "2.3", "2.4", "2.5", ""], + "3.14": ["2.3", "2.4", "2.5", ""], } # Deps that depend on numpy version (1.x vs 2.x path). @@ -46,7 +47,32 @@ "3.11": ["1.9", "1.10", "1.11", "1.12", "1.13", "1.14", ""], "3.12": ["1.11", "1.12", "1.13", "1.14", ""], }, - "2.x": ["1.13", "1.14", "1.15", "1.16", "1.17", ""], + "2.x": ["1.13", "1.14", "1.15", "1.16", "1.17", "1.18", ""], +} + +# Oldest Python each scipy pin supports (scipy's requires-python). The "2.x" pool is +# shared by every Python, so re-picks run through `_pick_scipy` to drop pins this +# Python has no build for. +# MAINT: 2026-09-16 scipy 1.18 requires Python >=3.12 +SCIPY_MIN_PYTHON = {"1.18": (3, 12)} + +# The numpy ceiling each scipy pin declares, the same idea as NUMBA_MAX_NUMPY below: +# scipy caps `numpy =2.5; -# see the awkward constraints in `apply_constraints`. +# which numpy 2.0 removed. Only >=2.10 pins survive numpy >=2.5; see the awkward +# constraints in `apply_constraints`. AWKWARD_VERSIONS = { "1.x": { "3.11": ["2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9", ""], "3.12": ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9", ""], }, - "2.x": ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", ""], + "2.x": ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12", "2.13", ""], } NUMBA_VERSIONS = { "1.x": ["0.57", "0.58", "0.59", "0.60", "0.61", ""], - "2.x": ["0.62", "0.63", "0.64", ""], + "2.x": ["0.62", "0.63", "0.64", "0.65", "0.66", "0.67", ""], } # Oldest numba that supports each Python. NUMBA_MIN_PYTHON = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} +# The numpy ceiling each numba pin declares. numba caps `numpy =1.15 needs numpy + >=1.26.4), so every re-pick site gets the same answer no matter where it sits in the + constraint order. "" (latest) is always a candidate: an unpinned scipy is one conda + resolves itself, so it satisfies every rule here by construction. + """ + return random.choice( + [ + s + for s in pool + if s + and _ver(s) >= floor + and SCIPY_MAX_NUMPY[s] > npver + and (_ver(s) < (1, 15) or npver >= (1, 26)) + and _ver(pyver) >= SCIPY_MIN_PYTHON.get(s, (0,)) + ] + + [""] + ) + + +def _numpy_ver(v, pyver): + """Version of numpy to compare against, resolving "" (latest) to what it installs. + + An unpinned numpy is the newest release this Python can actually get, which is the + newest pin in its pool, not something infinitely new: numba 0.67 caps numpy at <2.6 + and still qualifies today. Deriving it from the pool keeps it honest per Python + (py3.11 tops out at numpy 2.4) and leaves nothing extra to bump. + """ + if v["numpy"]: + return _ver(v["numpy"]) + return max(_ver(n) for n in NUMPY_VERSIONS[pyver] if n) + + def _min_for_python(table, pyver): """Look up `pyver` in a {python version: minimum} table, or None if it predates it. @@ -193,6 +289,10 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): The networkx section reads numpy, scipy, and pandas, so it runs after all three are final. """ + # Nothing below changes the numpy pick, so derive these once. + npver = _numpy_ver(v, pyver) + np_is_1x = v["numpy"].startswith("1.") if v["numpy"] else False + # --- scipy / numpy constraints --- # scipy >=1.15 requires numpy >=1.26.4 @@ -200,29 +300,23 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): candidates = [s for s in scipy_pool if s and _ver(s) < (1, 15)] v["scipy"] = random.choice(candidates) if candidates else "1.14" - # scipy <1.13 doesn't support numpy 2.x (safety net) - np_is_1x = v["numpy"].startswith("1.") if v["numpy"] else False - if not np_is_1x and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 13): - v["scipy"] = random.choice([s for s in scipy_pool if _ver(s) >= (1, 13)]) - - # scipy <1.15.1 requires numpy <2.3; scipy <1.16 requires numpy <2.5 - if _ver(v["numpy"]) >= (2, 3): - if v["scipy"] in ("1.13", "1.14"): - v["scipy"] = random.choice(["1.16", "1.17", ""]) - elif v["scipy"] == "1.15": - v["scipy"] = random.choice(["1.15", "1.16", "1.17", ""]) - - # numpy 1.26 + scipy 1.9 conflict - if v["numpy"] == "1.26" and v["scipy"] == "1.9": - v["scipy"] = random.choice(["1.10", "1.11", ""]) + # scipy's numpy ceiling (see SCIPY_MAX_NUMPY). This covers every scipy/numpy + # pairing at once: the pre-1.13 pins against numpy 2.x, 1.9 against numpy 1.26, + # and each later pin against the numpy that outgrew it. + if (ceiling := SCIPY_MAX_NUMPY.get(v["scipy"])) is not None and npver >= ceiling: + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool) # --- scipy / Python version availability --- # scipy <1.14 has no py3.13 builds; scipy <1.16 has no py3.14 builds if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): - v["scipy"] = random.choice(["1.16", "1.17", ""]) + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 16)) elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": - v["scipy"] = random.choice(["1.14", "1.15", "1.16", "1.17", ""]) + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 14)) + + # A scipy pin whose requires-python is newer than this Python has no build for it + if v["scipy"] and _ver(pyver) < SCIPY_MIN_PYTHON.get(v["scipy"], (0,)): + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool) # --- pandas constraints --- @@ -230,12 +324,10 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": v["pandas"] = random.choice(["2.3", "3.0", ""]) - # pandas 3.0 requires numba >=0.60 and scipy >=1.14.1 - if v["pandas"] == "3.0": - if v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 60): - v["numba"] = "0.60" - if v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): - v["scipy"] = random.choice(["1.15", "1.16", "1.17", ""]) + # pandas 3.0 requires scipy >=1.14.1. It also requires numba >=0.60, which needs no + # rule: pandas 3.0 only appears in the numpy 2.x pool, whose numba pins start at 0.62. + if v["pandas"] == "3.0" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 15)) # conda-forge pandas >=2.2 carries run_constrained "scipy >=1.10.0" (latest 2.2.3 # build and all 2.3 builds; 2.0/2.1 declare no scipy constraint), so a pandas @@ -300,12 +392,14 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- matplotlib / numpy floor (notebooks job) --- - # matplotlib is installed unpinned for the notebooks task. matplotlib 3.11.0's - # runtime check requires numpy >=1.25, but its conda-forge metadata does not, - # so the solver happily pairs matplotlib 3.11 with a numpy 1.24 pin and the - # notebooks then die at `import matplotlib`. Pin below 3.11 whenever the - # numpy pick sits under that floor; empty means latest is fine. - # MAINT: 2026-08-05 confirmed with matplotlib 3.11.0 + numpy 1.24.4 + # matplotlib is installed unpinned for the notebooks task. matplotlib 3.11's runtime + # check requires numpy >=1.25. conda-forge's 3.11.1 builds declare that floor, so the + # solver backs off on its own, but the 3.11.0 builds still say only "numpy >=1.23": + # the solver pairs 3.11.0 with a numpy 1.24 pin and the notebooks die at + # `import matplotlib`. Pin below 3.11 whenever the numpy pick sits under that floor; + # empty means latest is fine. + # MAINT: 2026-09-16 conda-forge matplotlib-base 3.11.0 depends on "numpy >=1.23", + # 3.11.1 on "numpy >=1.25" if v["numpy"] and _ver(v["numpy"]) < (1, 25): v["matplotlib"] = "<3.11" else: @@ -320,26 +414,15 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): pool = [n for n in numba_pool if _ver(n) >= min_ver] v["numba"] = random.choice(pool) if pool else "" - # numba <0.64 requires numpy <2.4 - if _ver(v["numpy"]) >= (2, 4) and v["numba"] in ("0.62", "0.63"): - v["numba"] = "0.64" - - # conda-forge numba 0.57 requires numpy <1.25, so that pairing fails the - # conda solve outright. Verified by dry-run solves: 0.57+1.25 conflicts; - # 0.58+1.25, 0.58+1.26, and 0.59+1.26 all solve. - # MAINT: 2026-08-05 confirmed against conda-forge with mamba dry-runs - if v["numba"] == "0.57" and np_is_1x and _ver(v["numpy"]) >= (1, 25): - v["numba"] = "0.58" - - # numba <0.62 doesn't support numpy 2.x - if not np_is_1x and v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 62): - v["numba"] = "NA" - - # --- sparse --- - - # sparse doesn't support Python 3.13+ - if _ver(pyver) >= (3, 13): - v["sparse"] = "NA" + # numba's numpy ceiling (see NUMBA_MAX_NUMPY). This covers every numba/numpy + # pairing rule at once: 0.57 against numpy >=1.25, the pre-0.62 pins against + # numpy 2.x, and each later pin against the numpy that outgrew it. Re-pick from + # the pins that clear both this numpy and the Python minimum, or skip numba. + ceiling = _numba_ceiling(v["numba"]) + if ceiling is not None and npver >= ceiling: + min_ver = _min_for_python(NUMBA_MIN_PYTHON, pyver) + pool = [n for n in numba_pool if n and NUMBA_MAX_NUMPY[n] > npver and _ver(n) >= min_ver] + v["numba"] = random.choice(pool) if pool else "NA" # --------------------------------------------------------------------------- @@ -392,7 +475,7 @@ def pick_versions(pyver, source_type): "numba": random.choice(numba_pool), "networkx": random.choice(NETWORKX_VERSIONS[pyver]), "pyyaml": random.choice(PYYAML_VERSIONS[pyver]), - "sparse": _pick_scalar_or_list(SPARSE_VERSIONS[pyver]), + "sparse": random.choice(SPARSE_VERSIONS[pyver]), } # Source builds have issues with some numpy/scipy/pandas versions; @@ -411,11 +494,6 @@ def pick_versions(pyver, source_type): return v -def _pick_scalar_or_list(pool): - """Handle pools that are either "NA" (string) or a list of choices.""" - return pool if isinstance(pool, str) else random.choice(pool) - - def _pick_psg(npver, pyver, source_type): """Pick python-suitesparse-graphblas version.""" if source_type == "upstream": @@ -430,8 +508,7 @@ def _pick_psg(npver, pyver, source_type): return "" return f"{eq}{random.choice(pool)}" pool = PSG_VERSIONS_NP2 - floors = _min_for_python(PSG_MIN_PYTHON, pyver) - if floors is not None: + if (floors := _min_for_python(PSG_MIN_PYTHON, pyver)) is not None: floor = _ver(floors[source_type]) pool = [ver for ver in pool if _ver(ver) >= floor] return random.choice([f"{eq}{ver}" for ver in pool] + [""]) @@ -507,7 +584,9 @@ def format_summary(v): "matplotlib", ): name = _SUMMARY_NAMES[key] - val = v[key] + # psg carries its own "=" or "==" pin prefix; strip it so the summary reads + # "psg=10.5.0.0" like every other entry instead of "psg==10.5.0.0". + val = v[key].lstrip("=") if key == "psg" else v[key] if val == "NA": parts.append(f"{name}=NA") elif val == "": @@ -522,6 +601,31 @@ def format_summary(v): # --------------------------------------------------------------------------- +def check_ceilings(): + """Check that every scipy and numba pin has a numpy-ceiling row. Returns errors. + + A pin with no row would silently skip the numpy-ceiling rule and be paired with a + numpy it refuses to import against. + """ + tables = [ + ( + "SCIPY_MAX_NUMPY", + "scipy", + SCIPY_MAX_NUMPY, + [SCIPY_VERSIONS["2.x"], *SCIPY_VERSIONS["1.x"].values()], + ), + ("NUMBA_MAX_NUMPY", "numba", NUMBA_MAX_NUMPY, list(NUMBA_VERSIONS.values())), + ] + errors = [] + for name, dep, table, pools in tables: + pins = {p for pool in pools for p in pool if p} + if missing := sorted(pins - table.keys(), key=_ver): + errors.append(f"{name} has no numpy ceiling for {dep} {', '.join(missing)}") + if extra := sorted(table.keys() - pins, key=_ver): + errors.append(f"{name} has rows for untested {dep} {', '.join(extra)}") + return errors + + def check_python_pools(): """Check that every per-Python table covers the same Pythons. Returns list of errors. @@ -555,34 +659,28 @@ def validate(v, pyver, source_type=None): if v["numpy"] in ("1.24", "1.25") and _ver(v["scipy"]) >= (1, 15): errors.append(f"scipy {v['scipy']} requires numpy >=1.26.4, got {v['numpy']}") - # scipy <1.13 requires numpy 1.x - if not np_is_1x and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 13): - errors.append(f"scipy {v['scipy']} doesn't support numpy 2.x") - - # scipy <1.15.1 requires numpy <2.3 - if _ver(v["numpy"]) >= (2, 3) and v["scipy"] in ("1.13", "1.14"): - errors.append(f"scipy {v['scipy']} requires numpy <2.3, got {v['numpy']}") - - # numpy 1.26 + scipy 1.9 - if v["numpy"] == "1.26" and v["scipy"] == "1.9": - errors.append("numpy 1.26 + scipy 1.9 conflict") + # scipy's declared numpy ceiling (see SCIPY_MAX_NUMPY) + ceiling = SCIPY_MAX_NUMPY.get(v["scipy"]) + if ceiling is not None and _numpy_ver(v, pyver) >= ceiling: + cap = ".".join(str(x) for x in ceiling) + errors.append(f"scipy {v['scipy']} requires numpy <{cap}, got {v['numpy'] or 'latest'}") # scipy Python availability if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): errors.append(f"scipy {v['scipy']} has no py{pyver} build") elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": errors.append(f"scipy 1.13 has no py{pyver} build") + if v["scipy"] and _ver(pyver) < (floor := SCIPY_MIN_PYTHON.get(v["scipy"], (0,))): + need = ".".join(str(x) for x in floor) + errors.append(f"scipy {v['scipy']} requires Python >={need}, has no py{pyver} build") # pandas Python availability if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": errors.append(f"pandas 2.2 has no py{pyver} build") - # pandas 3.0 requirements - if v["pandas"] == "3.0": - if v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 60): - errors.append(f"pandas 3.0 requires numba >=0.60, got {v['numba']}") - if v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): - errors.append(f"pandas 3.0 requires scipy >=1.14.1, got {v['scipy']}") + # pandas 3.0 requirements (numba >=0.60 is unreachable; see `apply_constraints`) + if v["pandas"] == "3.0" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): + errors.append(f"pandas 3.0 requires scipy >=1.14.1, got {v['scipy']}") # conda-forge pandas >=2.2 constrains scipy >=1.10.0 (fails the conda solve) if ( @@ -607,10 +705,6 @@ def validate(v, pyver, source_type=None): if v["numpy"] and _ver(v["numpy"]) < (1, 25) and v.get("matplotlib", "") != "<3.11": errors.append(f"matplotlib unpinned alongside numpy {v['numpy']} (needs <3.11)") - # conda-forge numba 0.57 requires numpy <1.25 (fails the conda solve) - if v["numba"] == "0.57" and v["numpy"] and _ver(v["numpy"]) >= (1, 25) and np_is_1x: - errors.append(f"numba 0.57 requires numpy <1.25, got {v['numpy']}") - # awkward <2.10 requires numpy <2.5 (generic-unit datetime64("NaT") at import) if ( _ver(v["numpy"]) >= (2, 5) @@ -635,17 +729,18 @@ def validate(v, pyver, source_type=None): if v["numba"] not in ("", "NA") and _ver(v["numba"]) < numba_min: errors.append(f"numba {v['numba']} doesn't support Python {pyver}") - # numba <0.64 requires numpy <2.4 - if v["numba"] in ("0.62", "0.63") and _ver(v["numpy"]) >= (2, 4): - errors.append(f"numba {v['numba']} requires numpy <2.4, got {v['numpy']}") - - # numba <0.62 requires numpy 1.x - if not np_is_1x and v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 62): - errors.append(f"numba {v['numba']} doesn't support numpy 2.x") + # numba's declared numpy ceiling (see NUMBA_MAX_NUMPY) + ceiling = _numba_ceiling(v["numba"]) + if ceiling is not None and _numpy_ver(v, pyver) >= ceiling: + cap = ".".join(str(x) for x in ceiling) + errors.append( + f"numba {v['numba'] or 'latest'} requires numpy <{cap}, " + f"got {v['numpy'] or 'latest'}" + ) - # sparse Python availability - if _ver(pyver) >= (3, 13) and v["sparse"] != "NA": - errors.append(f"sparse doesn't support Python {pyver}") + # sparse: the per-Python pool is the whole rule, so nothing may leave it + if v["sparse"] not in SPARSE_VERSIONS[pyver]: + errors.append(f"sparse {v['sparse']} isn't in the py{pyver} pool") # psg build availability (the floor depends on the Python and where psg comes from) floors = _min_for_python(PSG_MIN_PYTHON, pyver) @@ -699,9 +794,8 @@ def main(): if args.seed is not None: random.seed(args.seed) - pool_errors = check_python_pools() - if pool_errors: - print("Error: version pools disagree about which Pythons we support:", file=sys.stderr) + if pool_errors := check_python_pools() + check_ceilings(): + print("Error: version pools are inconsistent:", file=sys.stderr) for e in pool_errors: print(f" - {e}", file=sys.stderr) sys.exit(1)