diff --git a/.github/workflows/audit_versions.yml b/.github/workflows/audit_versions.yml new file mode 100644 index 000000000..646e96f59 --- /dev/null +++ b/.github/workflows/audit_versions.yml @@ -0,0 +1,80 @@ +# Checks the CI dependency pools in scripts/ci_pick_versions.py against the world. +# +# `ci_pick_versions.py --validate` runs in pre-commit and proves the picker agrees with +# the constraints it documents. It cannot notice when a constraint has drifted from +# reality, because that happens when a package releases, not when we commit. So this +# workflow runs weekly, and on changes to the picker or the audits themselves. +# +# A failure here does not mean CI is broken right now. It means a pool or a table no +# longer matches what PyPI or conda-forge publishes, and the next random pick that +# lands on the stale combination will fail for reasons that look unrelated. + +name: Audit dependency versions + +on: + schedule: + # Mondays at 06:00 UTC, before most of the week's PRs + - cron: "0 6 * * 1" + workflow_dispatch: + pull_request: + paths: + - scripts/ci_pick_versions.py + - scripts/audit_pypi_versions.py + - scripts/audit_conda_versions.py + - .github/workflows/audit_versions.yml + push: + branches: + - main + paths: + - scripts/ci_pick_versions.py + - scripts/audit_pypi_versions.py + - scripts/audit_conda_versions.py + - .github/workflows/audit_versions.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + pypi: + name: Audit against PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip packaging + - name: Stress-test the picker + run: python scripts/ci_pick_versions.py --validate + - name: Audit pools against PyPI + run: python scripts/audit_pypi_versions.py --samples 200 + + conda: + name: Audit against conda-forge + runs-on: ubuntu-latest + # conda-forge is what CI installs from, so this is the one that binds. It needs a + # conda to search with, which is why it is a separate, slower job. + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: "3.12" + channels: conda-forge + conda-remove-defaults: true + channel-priority: strict + activate-environment: audit + - run: python -m pip install --upgrade pip packaging + - name: Audit pools against conda-forge + run: python scripts/audit_conda_versions.py --samples 100 diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 69fab3770..574bcd3d8 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -145,7 +145,15 @@ jobs: # Pick random compatible dependency versions using the version picker script. # See scripts/ci_pick_versions.py for version pools and constraint logic. pyver=$(python -c "v='${{ steps.pyver.outputs.selected }}'.split('.');print(f'{v[0]}.{v[1]}')") - eval "$(python scripts/ci_pick_versions.py --python "$pyver" --source ${{ steps.sourcetype.outputs.selected }})" + # Assign first, then eval. `eval "$(cmd)"` reports eval's status, not cmd's, and + # this step's shell has no `-e`, so a picker that exits non-zero (a pool table it + # rejects, say) would leave every *ver variable unset and quietly install the + # latest of everything: a green run that tested nothing it claims to have tested. + if ! picks=$(python scripts/ci_pick_versions.py --python "$pyver" --source ${{ steps.sourcetype.outputs.selected }}) ; then + echo "::error::ci_pick_versions.py failed; refusing to install unpinned dependencies" + exit 1 + fi + eval "$picks" # Set up psg conda package variable (conda-forge installs via conda, others via pip). # For conda-forge, psg must always be installed; an empty psgver just means "latest". diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3582142d2..3b36390ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,23 +42,20 @@ repos: - id: name-tests-test args: ["--pytest-test-first"] - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.25 + rev: "0.26" hooks: - id: validate-pyproject name: Validate pyproject.toml # I don't yet trust ruff to do what autoflake does - repo: https://github.com/PyCQA/autoflake - rev: v2.3.3 + rev: v2.4.0 hooks: - id: autoflake args: [--in-place] # We can probably remove `isort` if we come to trust `ruff --fix`, # but we'll need to figure out the configuration to do this in `ruff` - repo: https://github.com/pycqa/isort - # 9.0 is still in alpha (9.0.0a3) and a pre-release pin defeats - # pre-commit.ci's autoupdate "avoid prereleases" logic; stay on the last - # stable 8.x until 9.0 GAs. - rev: 8.0.1 + rev: 9.0.1 hooks: - id: isort # Let's keep `pyupgrade` even though `ruff --fix` probably does most of it @@ -73,12 +70,12 @@ repos: - id: auto-walrus args: [--line-length, "100"] - repo: https://github.com/psf/black-pre-commit-mirror - rev: 26.3.1 + rev: 26.5.1 hooks: - id: black - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.13 + rev: v0.16.8 hooks: - id: ruff-check args: [--fix-only, --show-fixes] @@ -92,10 +89,10 @@ repos: additional_dependencies: &flake8_dependencies # These versions need updated manually - flake8==7.3.0 - - flake8-bugbear==25.11.29 + - flake8-bugbear==26.9.9 - flake8-simplify==0.30.0 - repo: https://github.com/codespell-project/codespell - rev: v2.4.2 + rev: v2.4.3 hooks: - id: codespell types_or: [python, markdown, rst, toml, yaml] @@ -103,7 +100,7 @@ repos: - tomli; python_version<'3.11' files: ^(graphblas|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.13 + rev: v0.16.8 hooks: - id: ruff-check # - id: ruff-format # we prefer black; uncomment to compare with ruff's formatter @@ -115,16 +112,18 @@ repos: # `pyroma` may help keep our package standards up to date if best practices change. # This is probably a "low value" check though and safe to remove if we want faster pre-commit. - repo: https://github.com/regebro/pyroma + # 5.1 is still in beta (5.1b2) and a pre-release pin defeats pre-commit.ci's + # autoupdate "avoid prereleases" logic; stay on 5.0.1 until 5.1 GAs. rev: "5.0.1" hooks: - id: pyroma args: [-n, "10", .] - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.11.0.1" + rev: "v0.11.0.1-1" hooks: - id: shellcheck - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.8.3 + rev: v3.9.7 hooks: - id: prettier args: [--prose-wrap=preserve] @@ -149,7 +148,7 @@ repos: hooks: - id: actionlint - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.37.2 + rev: 0.38.0 hooks: - id: check-dependabot - id: check-github-workflows @@ -163,7 +162,7 @@ repos: # hooks: # - id: zizmor - repo: https://github.com/pysentry/pysentry-pre-commit - rev: v0.4.6 + rev: v0.5.0 hooks: - id: pysentry args: ["--format=human", "--fail-on=low", "--sources=pypa,osv,pypi"] @@ -206,6 +205,15 @@ repos: | scripts/find_aiisms\.sh | \.pre-commit-config\.yaml )$ + # Stress-tests every combination the CI dependency picker can produce against + # the constraints it documents. ~3s for 160k combinations, and it only runs + # when the picker itself changes. + - id: ci-pick-versions + name: Validate CI dependency version picker + entry: python scripts/ci_pick_versions.py --validate + language: system + files: ^scripts/ci_pick_versions\.py$ + pass_filenames: false # Add `--hook-stage manual` to pre-commit command to run (very slow) # It's probably better (and faster!) to simply run `pylint graphblas/some/file.py` - id: pylint diff --git a/pyproject.toml b/pyproject.toml index 9fe272c98..cdf8867c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,14 @@ io = [ "python-graphblas[networkx,scipy]", "python-graphblas[numba]; python_version<'3.15'", # make optional where numba is not supported "awkward >=2.0", - "sparse >=0.14; python_version<'3.13'", # make optional, b/c sparse needs numba + # sparse needs numba, so it stays optional. numba gained py3.13 support in 0.61 and + # py3.14 in 0.63; sparse 0.16 is the first release that postdates py3.13, and 0.18 the + # first that postdates py3.14. + "sparse >=0.14; python_version<'3.13'", + "sparse >=0.16; python_version=='3.13'", + # Upper guard for the same reason as numba above: sparse depends on numba, so it has + # to drop out wherever numba does, not just trail it. + "sparse >=0.18; python_version>='3.14' and python_version<'3.15'", # fast-matrix-market is no longer maintained; last supported Python is 3.12 ] viz = ["python-graphblas[networkx,scipy]", "matplotlib >=3.6"] @@ -342,6 +349,7 @@ ignore = [ "PLR0911", # Too many return statements "PLR0912", # Too many branches "PLR0913", # Too many arguments to function call + "PLR0917", # Too many positional arguments (Stable since ruff 0.16; same call for PLR0913) "PLR0915", # Too many statements "PLR2004", # Magic number used in comparison, consider replacing magic with a constant variable "PLW0603", # Using the global statement to update ... is discouraged (Note: yeah, discouraged, but too strict) @@ -362,6 +370,7 @@ ignore = [ # Ignored categories "C90", # mccabe (Too strict, but maybe we should make things less complex) + "CPY", # flake8-copyright (Stable since ruff 0.16; we don't put a header in every file) "I", # isort (Should we replace `isort` with this?) "ANN", # flake8-annotations (We don't use annotations yet) "BLE", # flake8-blind-except (Maybe consider) @@ -394,6 +403,7 @@ ignore = [ "NPY002", # numba doesn't support rng generator yet ] "graphblas/core/utils.py" = ["PLE0302"] # `__set__` is used as a property +"graphblas/core/formatting.py" = ["ISC004"] # Adjacent literals form one HTML fragment "scripts/ci_pick_versions.py" = ["S311"] # random.choice is intentional for CI version selection "graphblas/ss/_core.py" = ["N999"] # We want _core.py to be underscopre # Allow useless expressions, assert, pickle, RNG, print, no docstring, and yoda in tests diff --git a/scripts/audit_conda_versions.py b/scripts/audit_conda_versions.py new file mode 100755 index 000000000..0d2fe4c16 --- /dev/null +++ b/scripts/audit_conda_versions.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python +"""Audit the CI dependency pools in `ci_pick_versions.py` against conda-forge. + +CI installs its dependencies from conda-forge, so this is the authoritative companion +to `audit_pypi_versions.py`. It can see two things PyPI cannot: + +* which Pythons a version was actually built for. conda-forge builds more than PyPI + ships (it has a py311 build of pyyaml 5.4.1 where PyPI's newest wheel is cp39), and + sometimes fewer, so "no wheel" and "no build" are different questions. +* `constrains` metadata (conda's run_constrained). Several rules in the picker exist + only because a package constrains something it does not depend on, which is what + makes a pinned pair fail to solve. + +Usage: + python scripts/audit_conda_versions.py [--samples N] + +Requires `conda` on PATH and network access. Slower than the PyPI audit: one +`conda search` per package, a few seconds each. Exits non-zero if any check fails. +See also scripts/check_versions.sh for the by-hand version of the same questions. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import random +import re +import shutil +import subprocess +import sys +from pathlib import Path + +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import InvalidVersion, Version + +_PICKER = Path(__file__).with_name("ci_pick_versions.py") +_spec = importlib.util.spec_from_file_location("ci_pick_versions", _PICKER) +picker = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(picker) + +# Our internal names to conda-forge package names +CONDA_NAMES = {"psg": "python-suitesparse-graphblas", "matplotlib": "matplotlib-base"} + +PER_PYTHON_POOLS = { + "numpy": picker.NUMPY_VERSIONS, + "networkx": picker.NETWORKX_VERSIONS, + "pyyaml": picker.PYYAML_VERSIONS, + "sparse": picker.SPARSE_VERSIONS, +} + +SAMPLED_DEPS = ("scipy", "pandas", "awkward", "numba", "networkx", "sparse", "pyyaml", "psg") + +# Constraints the picker hard-codes because they come from conda metadata alone. If one +# of these drifts, the matching rule in `apply_constraints` needs another look. +# Each entry is (dep, version pin, field, expected constraint). +DOCUMENTED_METADATA = [ + ("networkx", "3.6", "constrains", "numpy >=1.25"), + ("networkx", "3.6", "constrains", "scipy >=1.11.2"), + ("networkx", "3.6", "constrains", "pandas >=2.0"), + ("networkx", "3.5", "constrains", "numpy >=1.25"), + ("pandas", "2.3", "constrains", "scipy >=1.10.0"), + # matplotlib 3.11.0's metadata understates its numpy floor; 3.11.1 fixed it. The + # picker pins matplotlib <3.11 for old numpy because of the 3.11.0 builds. + ("matplotlib", "3.11.0", "depends", "numpy >=1.23"), + ("matplotlib", "3.11.1", "depends", "numpy >=1.25"), +] + + +# --------------------------------------------------------------------------- +# conda access +# --------------------------------------------------------------------------- + +_cache: dict[tuple[str, str], list] = {} + + +def conda_search(dep, floor): + """All conda-forge builds of `dep` at or above `floor`, cached per process.""" + if (dep, floor) not in _cache: + name = CONDA_NAMES.get(dep, dep) + spec = f"{name}[channel=conda-forge]>={floor}" + result = subprocess.run( + [_CONDA, "search", "--json", spec], + capture_output=True, + text=True, + check=False, + timeout=600, + ) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError: + raise SystemExit(f"conda search failed for {spec}:\n{result.stderr.strip()}") from None + if isinstance(payload, dict) and payload.get("error"): + raise SystemExit(f"conda search failed for {spec}: {payload['error']}") + _cache[dep, floor] = [build for builds in payload.values() for build in builds] + return _cache[dep, floor] + + +def builds_for(dep, pin, floor): + """Builds of `dep` matching a pin ("" means every build at or above `floor`).""" + return [ + b + for b in conda_search(dep, floor) + if not pin or b["version"] == pin or b["version"].startswith(f"{pin}.") + ] + + +def build_supports(build, pyver): + """Whether a conda build can be installed on `pyver`.""" + minor = int(pyver.split(".")[1]) + if match := re.search(r"py3(\d\d)", build["build"]): + return int(match.group(1)) == minor + constraints = [] + for dep in build.get("depends", []): + tokens = dep.split() + if tokens[0] != "python" or len(tokens) < 2: + continue + constraints.append(f"=={tokens[1]}" if tokens[1].endswith(".*") else tokens[1]) + if not constraints: + return True # noarch with a bare `python` dep + try: + return Version(pyver) in SpecifierSet(",".join(constraints)) + except (InvalidSpecifier, InvalidVersion): + return True + + +def supports_python(dep, pin, pyver, floor): + """Whether any build matching the pin supports `pyver`.""" + return any(build_supports(b, pyver) for b in builds_for(dep, pin, floor)) + + +def numpy_cap(dep, pin, floor): + """The tightest `numpy declared: + problems.append( + f"{dep} {pin} caps numpy at {declared} on conda-forge, " + f"but the table allows up to {ceiling}" + ) + return problems + + +def note_conservative_ceilings(): + """Ceiling rows stricter than conda-forge's cap. Advisory: coverage left on the table.""" + notes = [] + for dep, table in (("scipy", picker.SCIPY_MAX_NUMPY), ("numba", picker.NUMBA_MAX_NUMPY)): + floor = _oldest_pin(dep) + for pin, ceiling in table.items(): + declared = numpy_cap(dep, pin, floor) + if declared is not None and ceiling < declared: + notes.append( + f"{dep} {pin} is capped at {ceiling} in the table, " + f"but conda-forge allows numpy up to {declared}" + ) + return notes + + +def check_psg_python_floors(): + """PSG_MIN_PYTHON's conda-forge floor must be the oldest psg built for that Python.""" + problems = [] + floor = _oldest_pin("psg") + for pyver, sources in picker.PSG_MIN_PYTHON.items(): + expected = sources.get("conda-forge") + built = [p for p in picker.PSG_VERSIONS_NP2 if supports_python("psg", p, pyver, floor)] + if built and min(built, key=picker._ver) != expected: + problems.append( + f"PSG_MIN_PYTHON says conda-forge py{pyver} starts at psg {expected}, " + f"but {min(built, key=picker._ver)} has a py{pyver} build" + ) + return problems + + +def check_documented_metadata(): + """Constraints the picker's rules quote must still be what conda-forge publishes.""" + problems = [] + for dep, pin, field, expected in DOCUMENTED_METADATA: + matching = builds_for(dep, pin, _oldest_pin(dep)) + if not matching: + problems.append(f"{dep} {pin} has no conda-forge build; cannot confirm {expected!r}") + continue + if not any(expected in build.get(field, []) for build in matching): + problems.append(f"no {dep} {pin} build declares {field} {expected!r} any more") + return problems + + +def report_newer_releases(): + """conda-forge versions newer than our newest pin. Informational, never a failure.""" + news = [] + for dep in ( + "numpy", + "scipy", + "pandas", + "awkward", + "numba", + "networkx", + "pyyaml", + "sparse", + "psg", + ): + floor = _oldest_pin(dep) + newest_pin = max( + (b["version"] for b in builds_for(dep, "", floor)), key=_version_key, default=None + ) + tested = _newest_tested(dep) + if not newest_pin or not tested: + continue + # A pin names a series, so "2.5" already covers conda-forge's 2.5.3 + in_tested_series = newest_pin == tested or newest_pin.startswith(f"{tested}.") + if not in_tested_series and _version_key(newest_pin) > _version_key(tested): + news.append(f"{dep}: newest pin is {tested}, conda-forge has {newest_pin}") + return news + + +def _newest_tested(dep): + pools = { + "numpy": picker.NUMPY_VERSIONS, + "scipy": {"": picker.SCIPY_VERSIONS["2.x"]}, + "pandas": {"": picker.PANDAS_VERSIONS["2.x"]}, + "awkward": {"": picker.AWKWARD_VERSIONS["2.x"]}, + "numba": {"": picker.NUMBA_VERSIONS["2.x"]}, + "networkx": picker.NETWORKX_VERSIONS, + "pyyaml": picker.PYYAML_VERSIONS, + "sparse": picker.SPARSE_VERSIONS, + "psg": {"": picker.PSG_VERSIONS_NP2}, + }[dep] + pins = {p for group in pools.values() for p in group if p and p != "NA"} + return max(pins, key=picker._ver) if pins else None + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +_CONDA = shutil.which("conda") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--samples", + type=int, + default=100, + help="random picks per Python/source combination (default: 100)", + ) + parser.add_argument("--seed", type=int, default=None, help="seed for reproducible sampling") + args = parser.parse_args() + if _CONDA is None: + print("conda is not on PATH; this audit needs it", file=sys.stderr) + return 2 + if args.seed is not None: + random.seed(args.seed) + + checks = ( + ("per-Python pools", check_per_python_pools), + ("sampled picks", lambda: check_sampled_picks(args.samples)), + ("numpy ceiling tables", check_numpy_ceilings), + ("psg Python floors", check_psg_python_floors), + ("documented conda metadata", check_documented_metadata), + ) + failures = 0 + for name, check in checks: + problems = check() + if problems: + failures += len(problems) + print(f"FAIL {name}:") + for problem in problems: + print(f" - {problem}") + else: + print(f"ok {name}") + + if notes := note_conservative_ceilings(): + print("\nNotes (stricter than conda-forge needs; often right, since conda's depends") + print("carry ABI caps while a package's real runtime cap lives upstream):") + for note in notes: + print(f" - {note}") + + if news := report_newer_releases(): + print("\nNewer versions exist (not a failure; update the pools when convenient):") + for line in news: + print(f" - {line}") + + print(f"\n{failures} problem(s) found", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/audit_pypi_versions.py b/scripts/audit_pypi_versions.py new file mode 100755 index 000000000..75cc202e9 --- /dev/null +++ b/scripts/audit_pypi_versions.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python +"""Audit the CI dependency pools in `ci_pick_versions.py` against PyPI. + +`ci_pick_versions.py --validate` proves the picker agrees with the constraints it +documents. It cannot notice when a constraint has drifted from reality: a pin that +never had a build for a Python we test, or a ceiling table that a new release moved. +This script checks the pools and tables against what the packages actually declare. + +The picker's rules are written for conda-forge, which is what CI installs from, so +treat PyPI as a fast proxy and `audit_conda_versions.py` as the authority. Anything +this script reports is worth looking at in both places. + +Usage: + python scripts/audit_pypi_versions.py [--samples N] + +Requires network access and `packaging`. Exits non-zero if any check fails. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import random +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.version import InvalidVersion, Version + +# Import the picker as a module so the pools stay the single source of truth +_PICKER = Path(__file__).with_name("ci_pick_versions.py") +_spec = importlib.util.spec_from_file_location("ci_pick_versions", _PICKER) +picker = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(picker) + +# Our internal names to PyPI project names (conda-forge calls psg python-suitesparse-graphblas) +PYPI_NAMES = {"pyyaml": "PyYAML", "psg": "suitesparse-graphblas"} + +# Pools keyed by Python version, where every pin is offered on that Python and nowhere else +PER_PYTHON_POOLS = { + "numpy": picker.NUMPY_VERSIONS, + "networkx": picker.NETWORKX_VERSIONS, + "pyyaml": picker.PYYAML_VERSIONS, + "sparse": picker.SPARSE_VERSIONS, +} + +# Deps a pick can carry, in the order the picker fills them in +SAMPLED_DEPS = ("scipy", "pandas", "awkward", "numba", "networkx", "sparse", "pyyaml", "psg") + + +# --------------------------------------------------------------------------- +# PyPI access +# --------------------------------------------------------------------------- + +_cache: dict[str, dict] = {} + + +def _pypi(path): + """GET a PyPI JSON endpoint, cached for the life of the process.""" + if path not in _cache: + url = f"https://pypi.org/pypi/{path}/json" + request = urllib.request.Request(url, headers={"User-Agent": "python-graphblas-audit"}) + with urllib.request.urlopen(request, timeout=90) as response: # noqa: S310 + _cache[path] = json.load(response) + return _cache[path] + + +def releases(dep): + """Every non-prerelease, non-yanked release of `dep`: {Version: {...}}.""" + key = f"releases:{dep}" + if key not in _cache: + data = _pypi(PYPI_NAMES.get(dep, dep)) + out = {} + for raw, files in data["releases"].items(): + files = [f for f in files if not f["yanked"]] + if not files: + continue + try: + version = Version(raw) + except InvalidVersion: + continue + if version.is_prerelease: + continue + out[version] = { + "requires_python": files[0].get("requires_python"), + "cpython": {int(m) for f in files for m in re.findall(r"cp3(\d+)", f["filename"])}, + } + _cache[key] = out + return _cache[key] + + +def in_series(dep, pin): + """Every release matching a pin. An empty pin means "latest", so every release.""" + return [v for v in releases(dep) if not pin or str(v) == pin or str(v).startswith(f"{pin}.")] + + +def newest_in_series(dep, pin, pyver=None): + """The release a pin resolves to, preferring one that `pyver` can install. + + Returns (version, installable). When no release in the series supports `pyver` the + newest one is returned with installable=False, so callers can still read its + metadata while reporting the gap. + """ + found = in_series(dep, pin) + if not found: + return None, False + if pyver is not None: + usable = [v for v in found if supports_python(dep, v, pyver)] + if usable: + return max(usable), True + return max(found), False + return max(found), True + + +def supports_python(dep, version, pyver): + """Whether `version` of `dep` can be installed on `pyver`. + + Projects that ship compiled wheels are judged by their CPython tags, which is the + only signal that catches a release whose requires-python is wide but that simply + never built for this interpreter. Pure-Python projects are judged by requires-python. + """ + info = releases(dep)[version] + if info["cpython"]: + return int(pyver.split(".")[1]) in info["cpython"] + requires = info["requires_python"] + return not requires or Version(pyver) in SpecifierSet(requires) + + +def numpy_bound(dep, version, pyver): + """The numpy requirement `version` of `dep` declares for `pyver`, if any. + + A project may list numpy more than once (numba 0.65 declares both `numpy>=1.22` and + `numpy<2.5,>=1.22`), so combine every applicable clause rather than taking the first. + """ + data = _pypi(f"{PYPI_NAMES.get(dep, dep)}/{version}") + combined = None + for raw in data["info"].get("requires_dist") or []: + requirement = Requirement(raw) + if requirement.name.lower() != "numpy": + continue + marker = requirement.marker + if marker is None or marker.evaluate({"python_version": pyver, "extra": ""}): + combined = ( + requirement.specifier if combined is None else combined & requirement.specifier + ) + return combined + + +# --------------------------------------------------------------------------- +# Checks +# --------------------------------------------------------------------------- + + +def check_per_python_pools(): + """Every pin in a per-Python pool must exist on PyPI at all.""" + problems = [] + for dep, pools in PER_PYTHON_POOLS.items(): + for pyver, pins in pools.items(): + for pin in pins: + if pin == "NA": + continue + version, _ = newest_in_series(dep, pin) + if version is None: + problems.append(f"{dep} {pin} in the py{pyver} pool has no release on PyPI") + return problems + + +def note_pypi_python_gaps(): + """Pins with no PyPI build for a Python we offer them on. Advisory, not a failure. + + CI installs these from conda-forge, which builds more than PyPI ships: conda-forge + has a py311 build of pyyaml 5.4.1 where PyPI's newest wheel is cp39. Treat a note + here as a prompt to check `audit_conda_versions.py`, not as a broken pool. + """ + notes = [] + for dep, pools in PER_PYTHON_POOLS.items(): + for pyver, pins in pools.items(): + for pin in pins: + if pin == "NA": + continue + version, installable = newest_in_series(dep, pin, pyver) + if version is not None and not installable: + notes.append( + f"{dep} {pin or 'latest'} is offered on py{pyver}, " + f"but PyPI's {version} has no py{pyver} build" + ) + return notes + + +def check_ceiling_tables(): + """A ceiling row must never sit above the cap the release actually declares. + + A row above the real cap lets through a pairing that breaks; a row below it is + conservative, which is often deliberate, so that is reported as a note instead. + """ + problems = [] + tables = (("scipy", picker.SCIPY_MAX_NUMPY), ("numba", picker.NUMBA_MAX_NUMPY)) + for dep, table in tables: + for pin, ceiling in table.items(): + version, _ = newest_in_series(dep, pin) + if version is None: + problems.append(f"{dep} {pin} has a ceiling row but no release on PyPI") + continue + # Read the bound on the oldest Python we test, where markers are least likely + # to narrow it, then compare against the "< X.Y" the table records. + specifier = numpy_bound(dep, version, min(picker.NUMPY_VERSIONS, key=picker._ver)) + declared = _upper_bound(specifier) + if declared is not None and ceiling > declared: + problems.append( + f"{dep} {pin} ({version}) caps numpy at {declared}, " + f"but the table allows up to {ceiling}" + ) + return problems + + +def note_conservative_ceilings(): + """Ceiling rows stricter than PyPI's cap. Advisory: usually a conda-forge cap.""" + notes = [] + for dep, table in (("scipy", picker.SCIPY_MAX_NUMPY), ("numba", picker.NUMBA_MAX_NUMPY)): + for pin, ceiling in table.items(): + version, _ = newest_in_series(dep, pin) + if version is None: + continue + declared = _upper_bound( + numpy_bound(dep, version, min(picker.NUMPY_VERSIONS, key=picker._ver)) + ) + if declared is not None and ceiling < declared: + notes.append( + f"{dep} {pin} is capped at {ceiling} in the table, " + f"but PyPI's {version} allows numpy up to {declared}" + ) + return notes + + +def _upper_bound(specifier): + """The ` resolved and not str(latest).startswith(newest_pin): + news.append(f"{dep}: newest pin is {newest_pin} ({resolved}), PyPI has {latest}") + return news + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--samples", + type=int, + default=100, + help="random picks per Python/source combination (default: 100)", + ) + parser.add_argument("--seed", type=int, default=None, help="seed for reproducible sampling") + args = parser.parse_args() + if args.seed is not None: + random.seed(args.seed) + + checks = ( + ("per-Python pools", check_per_python_pools), + ("numpy ceiling tables", check_ceiling_tables), + ("Python floor tables", check_python_floor_tables), + ("sampled picks", lambda: check_sampled_picks(args.samples)), + ) + failures = 0 + for name, check in checks: + try: + problems = check() + except (urllib.error.URLError, TimeoutError) as exc: + print(f"SKIP {name}: PyPI unreachable ({exc})", file=sys.stderr) + continue + if problems: + failures += len(problems) + print(f"FAIL {name}:") + for problem in problems: + print(f" - {problem}") + else: + print(f"ok {name}") + + try: + notes = note_conservative_ceilings() + note_pypi_python_gaps() + news = report_newer_releases() + except (urllib.error.URLError, TimeoutError) as exc: + print(f"SKIP notes: PyPI unreachable ({exc})", file=sys.stderr) + notes = news = [] + + if notes: + print("\nNotes (PyPI only; conda-forge builds more, so check the conda audit):") + for note in notes: + print(f" - {note}") + + if news: + print("\nNewer releases exist (not a failure; update the pools when convenient):") + for line in news: + print(f" - {line}") + + print(f"\n{failures} problem(s) found", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_versions.sh b/scripts/check_versions.sh index 3977bd8a0..d9959556d 100755 --- a/scripts/check_versions.sh +++ b/scripts/check_versions.sh @@ -6,16 +6,23 @@ # # When updating versions throughout the repo (CI, pyproject.toml, pre-commit, etc.), # also update these version numbers to match the latest versions we currently test. -conda search 'flake8-bugbear[channel=conda-forge]>=25.11.29' +# +# scripts/audit_pypi_versions.py and scripts/audit_conda_versions.py ask the same +# questions automatically, and also check the pools and tables in ci_pick_versions.py +# against what is published. The "Audit dependency versions" workflow runs both weekly. +conda search 'flake8-bugbear[channel=conda-forge]>=26.9.9' conda search 'flake8-simplify[channel=conda-forge]>=0.30.0' -conda search 'numpy[channel=conda-forge]>=2.4' +conda search 'numpy[channel=conda-forge]>=2.5' conda search 'pandas[channel=conda-forge]>=3.0' -conda search 'scipy[channel=conda-forge]>=1.17' +conda search 'scipy[channel=conda-forge]>=1.18' conda search 'networkx[channel=conda-forge]>=3.6' -conda search 'awkward[channel=conda-forge]>=2.11' -conda search 'sparse[channel=conda-forge]>=0.15' +conda search 'awkward[channel=conda-forge]>=2.13' +conda search 'sparse[channel=conda-forge]>=0.19' # fast_matrix_market is deprecated (no longer maintained; last supported Python is 3.12) -conda search 'numba[channel=conda-forge]>=0.64' +conda search 'numba[channel=conda-forge]>=0.67' conda search 'pyyaml[channel=conda-forge]>=6.0' -conda search 'python-suitesparse-graphblas[channel=conda-forge]>=10.3.1' +# matplotlib is installed unpinned for the notebooks job; ci_pick_versions.py caps it +# below 3.11 when the numpy pick predates matplotlib's numpy >=1.25 runtime floor. +conda search 'matplotlib-base[channel=conda-forge]>=3.11' +conda search 'python-suitesparse-graphblas[channel=conda-forge]>=10.5.0' # conda search 'python[channel=conda-forge]>=3.11 *pypy*' diff --git a/scripts/ci_pick_versions.py b/scripts/ci_pick_versions.py index 738ef99a8..2fcfafc02 100755 --- a/scripts/ci_pick_versions.py +++ b/scripts/ci_pick_versions.py @@ -23,13 +23,21 @@ # "" means "latest" (no pin). "NA" means "don't install". # # When updating versions here, also update scripts/check_versions.sh +# +# To add a Python version (3.15, say): add it to `choices` in the "RNG for Python +# version" step of test_and_build.yml, give it a row in every table keyed by Python +# below (`check_python_pools` fails the run if you miss one), and revisit the floors +# in `apply_constraints`. Those floors are written as `_ver(pyver) >= (3, N)`, so a +# new Python inherits the previous one's floors instead of escaping them. That is +# enough to keep CI honest, but usually still too loose to install, so check each dep. # --------------------------------------------------------------------------- +# numpy 2.5 requires Python >=3.12, so py3.11 stops at 2.4. NUMPY_VERSIONS = { "3.11": ["1.24", "1.25", "1.26", "2.0", "2.1", "2.2", "2.3", "2.4", ""], - "3.12": ["1.26", "2.0", "2.1", "2.2", "2.3", "2.4", ""], - "3.13": ["2.1", "2.2", "2.3", "2.4", ""], - "3.14": ["2.3", "2.4", ""], + "3.12": ["1.26", "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", ""], + "3.13": ["2.1", "2.2", "2.3", "2.4", "2.5", ""], + "3.14": ["2.3", "2.4", "2.5", ""], } # Deps that depend on numpy version (1.x vs 2.x path). @@ -39,7 +47,32 @@ "3.11": ["1.9", "1.10", "1.11", "1.12", "1.13", "1.14", ""], "3.12": ["1.11", "1.12", "1.13", "1.14", ""], }, - "2.x": ["1.13", "1.14", "1.15", "1.16", "1.17", ""], + "2.x": ["1.13", "1.14", "1.15", "1.16", "1.17", "1.18", ""], +} + +# Oldest Python each scipy pin supports (scipy's requires-python). The "2.x" pool is +# shared by every Python, so re-picks run through `_pick_scipy` to drop pins this +# Python has no build for. +# MAINT: 2026-09-16 scipy 1.18 requires Python >=3.12 +SCIPY_MIN_PYTHON = {"1.18": (3, 12)} + +# The numpy ceiling each scipy pin declares, the same idea as NUMBA_MAX_NUMPY below: +# scipy caps `numpy =2.5; -# see the awkward constraints in `apply_constraints`. +# which numpy 2.0 removed. Only >=2.10 pins survive numpy >=2.5; see the awkward +# constraints in `apply_constraints`. AWKWARD_VERSIONS = { "1.x": { "3.11": ["2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9", ""], "3.12": ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9", ""], }, - "2.x": ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", ""], + "2.x": ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12", "2.13", ""], } NUMBA_VERSIONS = { "1.x": ["0.57", "0.58", "0.59", "0.60", "0.61", ""], - "2.x": ["0.62", "0.63", "0.64", ""], + "2.x": ["0.62", "0.63", "0.64", "0.65", "0.66", "0.67", ""], +} + +# Oldest numba that supports each Python. +NUMBA_MIN_PYTHON = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} + +# The numpy ceiling each numba pin declares. numba caps `numpy =3.9), so hold +# them to the wheel floor rather than test a build psg never claimed works. +# MAINT: 2026-09-11 checked conda-forge build strings and PyPI wheel tags for psg 9-10 +PSG_MIN_PYTHON = { + "3.14": {"conda-forge": "10.0.1.1", "wheel": "10.3.1.0", "source": "10.3.1.0"}, } @@ -131,6 +217,62 @@ def _ver(s): return tuple(int(x) for x in s.split(".")) +def _numba_ceiling(pin): + """The numpy ceiling a numba pin declares, or None when numba is skipped.""" + if pin == "NA": + return None + return max(NUMBA_MAX_NUMPY.values()) if pin == "" else NUMBA_MAX_NUMPY[pin] + + +def _pick_scipy(pyver, npver, pool, floor=(0,)): + """Pick a scipy pin at or above `floor` that this Python and this numpy can use. + + Filters the pool by the pin's requires-python (SCIPY_MIN_PYTHON), its numpy ceiling + (SCIPY_MAX_NUMPY) and the one numpy floor scipy imposes (>=1.15 needs numpy + >=1.26.4), so every re-pick site gets the same answer no matter where it sits in the + constraint order. "" (latest) is always a candidate: an unpinned scipy is one conda + resolves itself, so it satisfies every rule here by construction. + """ + return random.choice( + [ + s + for s in pool + if s + and _ver(s) >= floor + and SCIPY_MAX_NUMPY[s] > npver + and (_ver(s) < (1, 15) or npver >= (1, 26)) + and _ver(pyver) >= SCIPY_MIN_PYTHON.get(s, (0,)) + ] + + [""] + ) + + +def _numpy_ver(v, pyver): + """Version of numpy to compare against, resolving "" (latest) to what it installs. + + An unpinned numpy is the newest release this Python can actually get, which is the + newest pin in its pool, not something infinitely new: numba 0.67 caps numpy at <2.6 + and still qualifies today. Deriving it from the pool keeps it honest per Python + (py3.11 tops out at numpy 2.4) and leaves nothing extra to bump. + """ + if v["numpy"]: + return _ver(v["numpy"]) + return max(_ver(n) for n in NUMPY_VERSIONS[pyver] if n) + + +def _min_for_python(table, pyver): + """Look up `pyver` in a {python version: minimum} table, or None if it predates it. + + A Python newer than every row inherits the newest row: it needs at least as new a + build as the newest Python we have characterized. That guess is conservative rather + than correct, so give a new Python its own row once its real floors are known. + """ + if pyver in table: + return table[pyver] + newest = max(table, key=_ver) + return table[newest] if _ver(pyver) > _ver(newest) else None + + # --------------------------------------------------------------------------- # Constraints # --------------------------------------------------------------------------- @@ -147,6 +289,10 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): The networkx section reads numpy, scipy, and pandas, so it runs after all three are final. """ + # Nothing below changes the numpy pick, so derive these once. + npver = _numpy_ver(v, pyver) + np_is_1x = v["numpy"].startswith("1.") if v["numpy"] else False + # --- scipy / numpy constraints --- # scipy >=1.15 requires numpy >=1.26.4 @@ -154,42 +300,34 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): candidates = [s for s in scipy_pool if s and _ver(s) < (1, 15)] v["scipy"] = random.choice(candidates) if candidates else "1.14" - # scipy <1.13 doesn't support numpy 2.x (safety net) - np_is_1x = v["numpy"].startswith("1.") if v["numpy"] else False - if not np_is_1x and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 13): - v["scipy"] = random.choice([s for s in scipy_pool if _ver(s) >= (1, 13)]) - - # scipy <1.15.1 requires numpy <2.3; scipy <1.16 requires numpy <2.5 - if _ver(v["numpy"]) >= (2, 3): - if v["scipy"] in ("1.13", "1.14"): - v["scipy"] = random.choice(["1.16", "1.17", ""]) - elif v["scipy"] == "1.15": - v["scipy"] = random.choice(["1.15", "1.16", "1.17", ""]) - - # numpy 1.26 + scipy 1.9 conflict - if v["numpy"] == "1.26" and v["scipy"] == "1.9": - v["scipy"] = random.choice(["1.10", "1.11", ""]) + # scipy's numpy ceiling (see SCIPY_MAX_NUMPY). This covers every scipy/numpy + # pairing at once: the pre-1.13 pins against numpy 2.x, 1.9 against numpy 1.26, + # and each later pin against the numpy that outgrew it. + if (ceiling := SCIPY_MAX_NUMPY.get(v["scipy"])) is not None and npver >= ceiling: + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool) # --- scipy / Python version availability --- # scipy <1.14 has no py3.13 builds; scipy <1.16 has no py3.14 builds - if pyver == "3.14" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): - v["scipy"] = random.choice(["1.16", "1.17", ""]) - elif pyver == "3.13" and v["scipy"] == "1.13": - v["scipy"] = random.choice(["1.14", "1.15", "1.16", "1.17", ""]) + if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 16)) + elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 14)) + + # A scipy pin whose requires-python is newer than this Python has no build for it + if v["scipy"] and _ver(pyver) < SCIPY_MIN_PYTHON.get(v["scipy"], (0,)): + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool) # --- pandas constraints --- # pandas <2.3 has no py3.14 builds - if pyver == "3.14" and v["pandas"] == "2.2": + if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": v["pandas"] = random.choice(["2.3", "3.0", ""]) - # pandas 3.0 requires numba >=0.60 and scipy >=1.14.1 - if v["pandas"] == "3.0": - if v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 60): - v["numba"] = "0.60" - if v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): - v["scipy"] = random.choice(["1.15", "1.16", "1.17", ""]) + # pandas 3.0 requires scipy >=1.14.1. It also requires numba >=0.60, which needs no + # rule: pandas 3.0 only appears in the numpy 2.x pool, whose numba pins start at 0.62. + if v["pandas"] == "3.0" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 15)) # conda-forge pandas >=2.2 carries run_constrained "scipy >=1.10.0" (latest 2.2.3 # build and all 2.3 builds; 2.0/2.1 declare no scipy constraint), so a pandas @@ -218,9 +356,9 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- awkward / Python version availability --- # awkward <2.7 has no py3.13 builds; awkward <2.8 has no py3.14 builds - if pyver == "3.14" and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): + if _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): v["awkward"] = random.choice(["2.8", "2.9", ""]) - elif pyver == "3.13" and v["awkward"] == "2.6": + elif _ver(pyver) >= (3, 13) and v["awkward"] == "2.6": v["awkward"] = random.choice(["2.7", "2.8", "2.9", ""]) # --- awkward / numpy 2.5 deprecations --- @@ -254,12 +392,14 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- matplotlib / numpy floor (notebooks job) --- - # matplotlib is installed unpinned for the notebooks task. matplotlib 3.11.0's - # runtime check requires numpy >=1.25, but its conda-forge metadata does not, - # so the solver happily pairs matplotlib 3.11 with a numpy 1.24 pin and the - # notebooks then die at `import matplotlib`. Pin below 3.11 whenever the - # numpy pick sits under that floor; empty means latest is fine. - # MAINT: 2026-08-05 confirmed with matplotlib 3.11.0 + numpy 1.24.4 + # matplotlib is installed unpinned for the notebooks task. matplotlib 3.11's runtime + # check requires numpy >=1.25. conda-forge's 3.11.1 builds declare that floor, so the + # solver backs off on its own, but the 3.11.0 builds still say only "numpy >=1.23": + # the solver pairs 3.11.0 with a numpy 1.24 pin and the notebooks die at + # `import matplotlib`. Pin below 3.11 whenever the numpy pick sits under that floor; + # empty means latest is fine. + # MAINT: 2026-09-16 conda-forge matplotlib-base 3.11.0 depends on "numpy >=1.23", + # 3.11.1 on "numpy >=1.25" if v["numpy"] and _ver(v["numpy"]) < (1, 25): v["matplotlib"] = "<3.11" else: @@ -267,34 +407,22 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- numba constraints --- - # numba minimum by Python version: 0.59 for 3.12, 0.61 for 3.13, 0.63 for 3.14 - numba_min = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} + # numba minimum by Python version (see NUMBA_MIN_PYTHON) if v["numba"] not in ("", "NA"): - min_ver = numba_min[pyver] + min_ver = _min_for_python(NUMBA_MIN_PYTHON, pyver) if _ver(v["numba"]) < min_ver: pool = [n for n in numba_pool if _ver(n) >= min_ver] v["numba"] = random.choice(pool) if pool else "" - # numba <0.64 requires numpy <2.4 - if _ver(v["numpy"]) >= (2, 4) and v["numba"] in ("0.62", "0.63"): - v["numba"] = "0.64" - - # conda-forge numba 0.57 requires numpy <1.25, so that pairing fails the - # conda solve outright. Verified by dry-run solves: 0.57+1.25 conflicts; - # 0.58+1.25, 0.58+1.26, and 0.59+1.26 all solve. - # MAINT: 2026-08-05 confirmed against conda-forge with mamba dry-runs - if v["numba"] == "0.57" and np_is_1x and _ver(v["numpy"]) >= (1, 25): - v["numba"] = "0.58" - - # numba <0.62 doesn't support numpy 2.x - if not np_is_1x and v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 62): - v["numba"] = "NA" - - # --- sparse --- - - # sparse doesn't support Python 3.13+ - if pyver in ("3.13", "3.14"): - v["sparse"] = "NA" + # numba's numpy ceiling (see NUMBA_MAX_NUMPY). This covers every numba/numpy + # pairing rule at once: 0.57 against numpy >=1.25, the pre-0.62 pins against + # numpy 2.x, and each later pin against the numpy that outgrew it. Re-pick from + # the pins that clear both this numpy and the Python minimum, or skip numba. + ceiling = _numba_ceiling(v["numba"]) + if ceiling is not None and npver >= ceiling: + min_ver = _min_for_python(NUMBA_MIN_PYTHON, pyver) + pool = [n for n in numba_pool if n and NUMBA_MAX_NUMPY[n] > npver and _ver(n) >= min_ver] + v["numba"] = random.choice(pool) if pool else "NA" # --------------------------------------------------------------------------- @@ -347,7 +475,7 @@ def pick_versions(pyver, source_type): "numba": random.choice(numba_pool), "networkx": random.choice(NETWORKX_VERSIONS[pyver]), "pyyaml": random.choice(PYYAML_VERSIONS[pyver]), - "sparse": _pick_scalar_or_list(SPARSE_VERSIONS[pyver]), + "sparse": random.choice(SPARSE_VERSIONS[pyver]), } # Source builds have issues with some numpy/scipy/pandas versions; @@ -366,11 +494,6 @@ def pick_versions(pyver, source_type): return v -def _pick_scalar_or_list(pool): - """Handle pools that are either "NA" (string) or a list of choices.""" - return pool if isinstance(pool, str) else random.choice(pool) - - def _pick_psg(npver, pyver, source_type): """Pick python-suitesparse-graphblas version.""" if source_type == "upstream": @@ -384,7 +507,10 @@ def _pick_psg(npver, pyver, source_type): if not pool: return "" return f"{eq}{random.choice(pool)}" - pool = PSG_VERSIONS_NP2["py314_only"] if pyver == "3.14" else PSG_VERSIONS_NP2["no_py314"] + pool = PSG_VERSIONS_NP2 + if (floors := _min_for_python(PSG_MIN_PYTHON, pyver)) is not None: + floor = _ver(floors[source_type]) + pool = [ver for ver in pool if _ver(ver) >= floor] return random.choice([f"{eq}{ver}" for ver in pool] + [""]) @@ -458,7 +584,9 @@ def format_summary(v): "matplotlib", ): name = _SUMMARY_NAMES[key] - val = v[key] + # psg carries its own "=" or "==" pin prefix; strip it so the summary reads + # "psg=10.5.0.0" like every other entry instead of "psg==10.5.0.0". + val = v[key].lstrip("=") if key == "psg" else v[key] if val == "NA": parts.append(f"{name}=NA") elif val == "": @@ -473,7 +601,56 @@ def format_summary(v): # --------------------------------------------------------------------------- -def validate(v, pyver): +def check_ceilings(): + """Check that every scipy and numba pin has a numpy-ceiling row. Returns errors. + + A pin with no row would silently skip the numpy-ceiling rule and be paired with a + numpy it refuses to import against. + """ + tables = [ + ( + "SCIPY_MAX_NUMPY", + "scipy", + SCIPY_MAX_NUMPY, + [SCIPY_VERSIONS["2.x"], *SCIPY_VERSIONS["1.x"].values()], + ), + ("NUMBA_MAX_NUMPY", "numba", NUMBA_MAX_NUMPY, list(NUMBA_VERSIONS.values())), + ] + errors = [] + for name, dep, table, pools in tables: + pins = {p for pool in pools for p in pool if p} + if missing := sorted(pins - table.keys(), key=_ver): + errors.append(f"{name} has no numpy ceiling for {dep} {', '.join(missing)}") + if extra := sorted(table.keys() - pins, key=_ver): + errors.append(f"{name} has rows for untested {dep} {', '.join(extra)}") + return errors + + +def check_python_pools(): + """Check that every per-Python table covers the same Pythons. Returns list of errors. + + Adding a Python to some tables but not others would otherwise surface as a KeyError + from somewhere deep in the picking, or (for the tables consulted only by a + constraint) as a silently wrong pick. + """ + tables = { + "NETWORKX_VERSIONS": NETWORKX_VERSIONS, + "PYYAML_VERSIONS": PYYAML_VERSIONS, + "SPARSE_VERSIONS": SPARSE_VERSIONS, + "NUMBA_MIN_PYTHON": NUMBA_MIN_PYTHON, + } + errors = [] + for name, table in tables.items(): + missing = sorted(NUMPY_VERSIONS.keys() - table.keys(), key=_ver) + extra = sorted(table.keys() - NUMPY_VERSIONS.keys(), key=_ver) + if missing: + errors.append(f"{name} has no entry for Python {', '.join(missing)}") + if extra: + errors.append(f"{name} has entries for unsupported Python {', '.join(extra)}") + return errors + + +def validate(v, pyver, source_type=None): """Check that a version combination satisfies all constraints. Returns list of errors.""" errors = [] np_is_1x = v["numpy"].startswith("1.") if v["numpy"] else False @@ -482,34 +659,28 @@ def validate(v, pyver): if v["numpy"] in ("1.24", "1.25") and _ver(v["scipy"]) >= (1, 15): errors.append(f"scipy {v['scipy']} requires numpy >=1.26.4, got {v['numpy']}") - # scipy <1.13 requires numpy 1.x - if not np_is_1x and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 13): - errors.append(f"scipy {v['scipy']} doesn't support numpy 2.x") - - # scipy <1.15.1 requires numpy <2.3 - if _ver(v["numpy"]) >= (2, 3) and v["scipy"] in ("1.13", "1.14"): - errors.append(f"scipy {v['scipy']} requires numpy <2.3, got {v['numpy']}") - - # numpy 1.26 + scipy 1.9 - if v["numpy"] == "1.26" and v["scipy"] == "1.9": - errors.append("numpy 1.26 + scipy 1.9 conflict") + # scipy's declared numpy ceiling (see SCIPY_MAX_NUMPY) + ceiling = SCIPY_MAX_NUMPY.get(v["scipy"]) + if ceiling is not None and _numpy_ver(v, pyver) >= ceiling: + cap = ".".join(str(x) for x in ceiling) + errors.append(f"scipy {v['scipy']} requires numpy <{cap}, got {v['numpy'] or 'latest'}") # scipy Python availability - if pyver == "3.14" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): - errors.append(f"scipy {v['scipy']} has no py3.14 build") - if pyver == "3.13" and v["scipy"] == "1.13": - errors.append("scipy 1.13 has no py3.13 build") + if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): + errors.append(f"scipy {v['scipy']} has no py{pyver} build") + elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": + errors.append(f"scipy 1.13 has no py{pyver} build") + if v["scipy"] and _ver(pyver) < (floor := SCIPY_MIN_PYTHON.get(v["scipy"], (0,))): + need = ".".join(str(x) for x in floor) + errors.append(f"scipy {v['scipy']} requires Python >={need}, has no py{pyver} build") # pandas Python availability - if pyver == "3.14" and v["pandas"] == "2.2": - errors.append("pandas 2.2 has no py3.14 build") + if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": + errors.append(f"pandas 2.2 has no py{pyver} build") - # pandas 3.0 requirements - if v["pandas"] == "3.0": - if v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 60): - errors.append(f"pandas 3.0 requires numba >=0.60, got {v['numba']}") - if v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): - errors.append(f"pandas 3.0 requires scipy >=1.14.1, got {v['scipy']}") + # pandas 3.0 requirements (numba >=0.60 is unreachable; see `apply_constraints`) + if v["pandas"] == "3.0" and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 15): + errors.append(f"pandas 3.0 requires scipy >=1.14.1, got {v['scipy']}") # conda-forge pandas >=2.2 constrains scipy >=1.10.0 (fails the conda solve) if ( @@ -521,10 +692,10 @@ def validate(v, pyver): errors.append(f"pandas {v['pandas']} constrains scipy >=1.10.0, got {v['scipy']}") # awkward Python availability - if pyver == "3.14" and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): - errors.append(f"awkward {v['awkward']} has no py3.14 build") - if pyver == "3.13" and v["awkward"] == "2.6": - errors.append("awkward 2.6 has no py3.13 build") + if _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): + errors.append(f"awkward {v['awkward']} has no py{pyver} build") + elif _ver(pyver) >= (3, 13) and v["awkward"] == "2.6": + errors.append(f"awkward 2.6 has no py{pyver} build") # awkward <2.6 requires numpy 1.x (numpy.AxisError) if not np_is_1x and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 6): @@ -534,10 +705,6 @@ def validate(v, pyver): if v["numpy"] and _ver(v["numpy"]) < (1, 25) and v.get("matplotlib", "") != "<3.11": errors.append(f"matplotlib unpinned alongside numpy {v['numpy']} (needs <3.11)") - # conda-forge numba 0.57 requires numpy <1.25 (fails the conda solve) - if v["numba"] == "0.57" and v["numpy"] and _ver(v["numpy"]) >= (1, 25) and np_is_1x: - errors.append(f"numba 0.57 requires numpy <1.25, got {v['numpy']}") - # awkward <2.10 requires numpy <2.5 (generic-unit datetime64("NaT") at import) if ( _ver(v["numpy"]) >= (2, 5) @@ -558,21 +725,30 @@ def validate(v, pyver): errors.append(f"networkx {nx} requires pandas >=2.0, got {v['pandas']}") # numba Python minimums - numba_min = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} - if v["numba"] not in ("", "NA") and _ver(v["numba"]) < numba_min[pyver]: + numba_min = _min_for_python(NUMBA_MIN_PYTHON, pyver) + if v["numba"] not in ("", "NA") and _ver(v["numba"]) < numba_min: errors.append(f"numba {v['numba']} doesn't support Python {pyver}") - # numba <0.64 requires numpy <2.4 - if v["numba"] in ("0.62", "0.63") and _ver(v["numpy"]) >= (2, 4): - errors.append(f"numba {v['numba']} requires numpy <2.4, got {v['numpy']}") - - # numba <0.62 requires numpy 1.x - if not np_is_1x and v["numba"] not in ("", "NA") and _ver(v["numba"]) < (0, 62): - errors.append(f"numba {v['numba']} doesn't support numpy 2.x") - - # sparse Python availability - if pyver in ("3.13", "3.14") and v["sparse"] != "NA": - errors.append(f"sparse doesn't support Python {pyver}") + # numba's declared numpy ceiling (see NUMBA_MAX_NUMPY) + ceiling = _numba_ceiling(v["numba"]) + if ceiling is not None and _numpy_ver(v, pyver) >= ceiling: + cap = ".".join(str(x) for x in ceiling) + errors.append( + f"numba {v['numba'] or 'latest'} requires numpy <{cap}, " + f"got {v['numpy'] or 'latest'}" + ) + + # sparse: the per-Python pool is the whole rule, so nothing may leave it + if v["sparse"] not in SPARSE_VERSIONS[pyver]: + errors.append(f"sparse {v['sparse']} isn't in the py{pyver} pool") + + # psg build availability (the floor depends on the Python and where psg comes from) + floors = _min_for_python(PSG_MIN_PYTHON, pyver) + if floors is not None and source_type in floors and v.get("psg"): + psgver = v["psg"].lstrip("=") + floor = floors[source_type] + if _ver(psgver) < _ver(floor): + errors.append(f"psg {psgver} has no py{pyver} {source_type} build (needs >={floor})") return errors @@ -585,7 +761,7 @@ def stress_test(n=10000): for source in ("conda-forge", "wheel", "source", "upstream"): for _ in range(n): v = pick_versions(pyver, source) - errs = validate(v, pyver) + errs = validate(v, pyver, source) total += 1 if errs: failures += 1 @@ -618,6 +794,12 @@ def main(): if args.seed is not None: random.seed(args.seed) + if pool_errors := check_python_pools() + check_ceilings(): + print("Error: version pools are inconsistent:", file=sys.stderr) + for e in pool_errors: + print(f" - {e}", file=sys.stderr) + sys.exit(1) + if args.validate: failures = stress_test() sys.exit(1 if failures else 0)