diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace4600a..94e8d4b16 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,12 @@ updates: directory: "/" schedule: interval: "weekly" + # Wait a week after a release before proposing it, so a compromised or + # broken release has time to be noticed and yanked. + cooldown: + default-days: 7 + # One PR per week for all action bumps instead of one PR per action. + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/audit_versions.yml b/.github/workflows/audit_versions.yml index 646e96f59..162713824 100644 --- a/.github/workflows/audit_versions.yml +++ b/.github/workflows/audit_versions.yml @@ -43,10 +43,10 @@ jobs: name: Audit against PyPI runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - run: python -m pip install --upgrade pip packaging @@ -64,10 +64,10 @@ jobs: run: shell: bash -el {0} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 with: auto-update-conda: true python-version: "3.12" diff --git a/.github/workflows/debug.yml b/.github/workflows/debug.yml index 45401c704..bb8503570 100644 --- a/.github/workflows/debug.yml +++ b/.github/workflows/debug.yml @@ -9,6 +9,9 @@ on: required: false default: false +permissions: + contents: read + jobs: build: runs-on: ubuntu-latest @@ -26,7 +29,7 @@ jobs: # - "conda-forge" steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -71,5 +74,5 @@ jobs: ( for attr in Matrix Scalar Vector Recorder _agg agg base binary descriptor dtypes exceptions expr ffi formatting infix init io lib mask matrix monoid op operator scalar semiring tests unary vector recorder _ss ss ; do echo python -c \"from graphblas import $attr\" ; if ! python -c "from graphblas import $attr" ; then exit 1 ; fi ; done ) - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 + uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3.24 if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }} diff --git a/.github/workflows/imports.yml b/.github/workflows/imports.yml index 7f1596c0a..9adfb6b0e 100644 --- a/.github/workflows/imports.yml +++ b/.github/workflows/imports.yml @@ -6,6 +6,14 @@ on: branches: - main +permissions: + contents: read + +# Cancels a run's identical twin only; see the note in test_and_build.yml. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.event.pull_request.head.sha || github.sha }} + cancel-in-progress: true + jobs: rngs: runs-on: ubuntu-latest @@ -25,7 +33,7 @@ jobs: - name: RNG for Python version id: pyver run: | - choices=(3.11 3.12 3.13 3.14) + choices=(3.11 3.12 3.13 3.14 3.15) selected=${choices[$((RANDOM % ${#choices[@]}))]} echo "selected: $selected (from: ${choices[*]})" echo "selected=$selected" >> "$GITHUB_OUTPUT" @@ -35,14 +43,14 @@ jobs: # runs-on: ${{ matrix.os }} # strategy: # matrix: - # python-version: ["3.11", "3.12", "3.13", "3.14"] + # python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"] # os: ["ubuntu-latest", "macos-latest", "windows-latest"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ needs.rngs.outputs.pyver }} # python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8bceeeadd..567dceb83 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,11 +16,11 @@ jobs: name: pre-commit-hooks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - - uses: pre-commit/action@v3.0.1 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index ad9284f66..fd2f3af10 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -5,6 +5,9 @@ on: tags: - "20*" +permissions: + contents: read + jobs: build_and_deploy: runs-on: ubuntu-latest @@ -13,12 +16,12 @@ jobs: shell: bash -l {0} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Install build dependencies @@ -27,16 +30,28 @@ jobs: python -m pip install build twine - name: Build wheel and sdist run: python -m build --sdist --wheel - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: releases path: dist if-no-files-found: error - name: Check with twine run: python -m twine check --strict dist/* - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.13.0 - if: startsWith(github.ref, 'refs/tags/') && github.repository == 'python-graphblas/python-graphblas' + + publish: + needs: build_and_deploy + if: startsWith(github.ref, 'refs/tags/') && github.repository == 'python-graphblas/python-graphblas' + runs-on: ubuntu-latest + # The trusted publisher registered on PyPI names this environment, which also asks a + # maintainer to approve the upload. + environment: pypi + permissions: + # Only this job can mint the OIDC token that PyPI exchanges for upload rights. + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} + name: releases + path: dist + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 000000000..e7840ac2c --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,58 @@ +name: Security + +# Run the security hooks of .pre-commit-config.yaml, with its versions and arguments, where +# pre-commit.ci cannot: with network access, and weekly, for advisories published since. +on: + pull_request: + paths: + - pyproject.toml + - .pre-commit-config.yaml + - .github/** + schedule: + # Mondays at 06:41 UTC + - cron: "41 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + pysentry: + name: pysentry + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + # Advisories against the floors in pyproject.toml + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + with: + extra_args: pysentry --all-files + + zizmor: + name: zizmor + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + # With a token, zizmor also runs its online audits, such as advisories against the actions + # these workflows use (known-vulnerable-actions) and pins to commits outside an action's + # repository (impostor-commit). + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + env: + GH_TOKEN: ${{ github.token }} + with: + extra_args: zizmor --all-files diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 8dec76f13..dadb23e36 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -17,9 +17,17 @@ on: branches: - main -# concurrency: -# group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} -# cancel-in-progress: true +permissions: + contents: read + +# GitHub can deliver a PR's pull_request event twice for one push when the PR's base +# branch moved in the same push, as it does for every PR above the bottom of a restacked +# stack. Keyed on the PR and its head commit, the later run cancels its identical twin, +# and a run for an earlier commit is left to finish, as before (a group keyed on the +# branch would cancel it). Pushes to main key on the run id, so none is ever cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.event.pull_request.head.sha || github.sha }} + cancel-in-progress: true jobs: rngs: @@ -100,12 +108,14 @@ jobs: # - os: ubuntu-24.04-arm # Linux aarch64 # slowtask: notebooks env: - # Wheels on OS X come with an OpenMP that conflicts with OpenMP from conda-forge. - # Setting this is a workaround. + # Wheels on OS X come with an OpenMP that conflicts with OpenMP from conda-forge: + # libomp aborts when a second copy starts in the process, unless this is set. Two + # copies can still deadlock or crash in an OpenMP barrier, so "Update env" keeps + # conda-forge's libomp out of macOS wheel draws where it can. KMP_DUPLICATE_LIB_OK: ${{ contains(matrix.os, 'macos') && 'TRUE' || 'FALSE' }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -117,7 +127,7 @@ jobs: # We may be able to support pypy if anybody asks for it # 3.9.16 0_73_pypy run: | - choices=(3.11 3.12 3.13 3.14) + choices=(3.11 3.12 3.13 3.14 3.15) selected=${choices[$((RANDOM % ${#choices[@]}))]} echo "selected: $selected (from: ${choices[*]})" echo "selected=$selected" >> "$GITHUB_OUTPUT" @@ -130,7 +140,7 @@ jobs: echo "selected: $selected (from: ${choices[*]})" echo "selected=$selected" >> "$GITHUB_OUTPUT" - name: Setup conda - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 id: setup_conda with: auto-update-conda: true @@ -139,17 +149,25 @@ jobs: conda-remove-defaults: ${{ contains(steps.pyver.outputs.selected, 'pypy') && 'false' || 'true' }} channel-priority: ${{ contains(steps.pyver.outputs.selected, 'pypy') && 'flexible' || 'strict' }} activate-environment: graphblas - auto-activate-base: false + # setup-miniconda deprecated ``auto-activate-base`` for this name, and warns + # on every run that still uses the old one. + auto-activate: false - name: Update env + id: update_env + env: + # Step outputs reach the script through the environment, not ``${{ }}``, so + # bash never parses them as code (zizmor's template-injection audit). + PYVER: ${{ steps.pyver.outputs.selected }} + SOURCETYPE: ${{ steps.sourcetype.outputs.selected }} run: | # Pick random compatible dependency versions using the version picker script. # See scripts/ci_pick_versions.py for version pools and constraint logic. - pyver=$(python -c "v='${{ steps.pyver.outputs.selected }}'.split('.');print(f'{v[0]}.{v[1]}')") + pyver=$(python -c "import os; v=os.environ['PYVER'].split('.');print(f'{v[0]}.{v[1]}')") # Assign first, then eval. `eval "$(cmd)"` reports eval's status, not cmd's, and # this step's shell has no `-e`, so a picker that exits non-zero (a pool table it # rejects, say) would leave every *ver variable unset and quietly install the # latest of everything: a green run that tested nothing it claims to have tested. - if ! picks=$(python scripts/ci_pick_versions.py --python "$pyver" --source ${{ steps.sourcetype.outputs.selected }}) ; then + if ! picks=$(python scripts/ci_pick_versions.py --python "$pyver" --source "$SOURCETYPE") ; then echo "::error::ci_pick_versions.py failed; refusing to install unpinned dependencies" exit 1 fi @@ -158,9 +176,44 @@ jobs: # Set up psg conda package variable (conda-forge installs via conda, others via pip). # For conda-forge, psg must always be installed; an empty psgver just means "latest". psg="" - if [[ ${{ steps.sourcetype.outputs.selected}} == "conda-forge" ]] ; then + if [[ $SOURCETYPE == "conda-forge" ]] ; then psg=python-suitesparse-graphblas${psgver} fi + # Wheel and source draws install psg in "Build extension module", where this + # step's shell variables no longer exist, so psgver goes there as an output. + echo "psgver=$psgver" >> "$GITHUB_OUTPUT" + + # Wheels bundle their own GraphBLAS. Otherwise give a pinned psg the GraphBLAS it was + # built for: python-suitesparse-graphblas X.Y.Z.W wraps SuiteSparse:GraphBLAS X.Y.Z, + # and psg's test_version_tracks_libgraphblas checks that they match. conda-forge's + # psg X.Y.Z.W accepts any graphblas >=X.Y.Z,> "$GITHUB_ENV" + fi # Decide whether to install numba and sparse. # Random skip: pypy always, 20% on Windows, 40% on other OSes (unless notebooks). @@ -203,6 +256,22 @@ jobs: echo "versions: np${npver} sp${spver} pd${pdver} ak${akver} nx${nxver} numba${numbaver} yaml${yamlver} sparse${sparsever} psg${psgver}" set -x # echo on + # conda-forge ships two ABIs for CPython 3.13+, and this step did not + # restate ``python``, so the solver was free to replace the interpreter + # setup-miniconda installed: a 3.14 draw on ubuntu-24.04-arm became + # 3.14.7-*_cp314t, where pandas 2.3 raises a GIL RuntimeWarning that + # filterwarnings=error turns into a collection error. Restating the + # version pins the interpreter and ``python-gil`` pins python_abi to + # the GIL build; setup-miniconda's own ``python-version`` above is + # equally ABI-free, so this covers that step too. + # python-suitesparse-graphblas pins the same pair in + # .github/workflows/test.yml, and asks for python-freethreading on the + # matrix entries that want the other ABI. + if [[ $PYVER == *pypy* ]] ; then + interpreter=pypy + else + interpreter="python=$PYVER python-gil" + fi $(command -v mamba || command -v conda) install -c nodefaults \ packaging pip pytest coverage pytest-randomly cffi donfig tomli c-compiler make \ pyyaml${yamlver} ${sparse} pandas${pdver} scipy${spver} numpy${npver} ${awkward} \ @@ -210,29 +279,32 @@ jobs: ${{ matrix.slowtask == 'pytest_bizarro' && 'black' || '' }} \ ${{ matrix.slowtask == 'notebooks' && '"matplotlib${mplver}" nbconvert jupyter "ipython>=7" drawsvg' || '' }} \ ${{ steps.sourcetype.outputs.selected == 'upstream' && 'cython' || '' }} \ - ${{ steps.sourcetype.outputs.selected != 'wheel' && '"graphblas>=7.4"' || '' }} \ - ${{ contains(steps.pyver.outputs.selected, 'pypy') && 'pypy' || '' }} \ + ${graphblas} ${blas:+"$blas"} \ + ${interpreter} \ ${{ matrix.os == 'windows-latest' && 'cmake' || 'm4' }} \ # ${{ matrix.os != 'windows-latest' && 'pytest-forked' || '' }} # to investigate crashes - name: Build extension module + env: + SOURCETYPE: ${{ steps.sourcetype.outputs.selected }} + PSGVER: ${{ steps.update_env.outputs.psgver }} run: | # ``python -m pip`` ensures the conda env's interpreter drives the # install, not a system ``pip`` on PATH. - if [[ ${{ steps.sourcetype.outputs.selected }} == "wheel" ]]; then + if [[ $SOURCETYPE == "wheel" ]]; then # Add --pre if installing a pre-release - python -m pip install --no-deps --only-binary ":all:" suitesparse-graphblas${psgver} + python -m pip install --no-deps --only-binary ":all:" "suitesparse-graphblas${PSGVER}" # Add the below line to the conda install command above if installing from test.pypi.org # ${{ steps.sourcetype.outputs.selected == 'wheel' && 'setuptools setuptools-git-versioning wheel cython' || '' }} \ # python -m pip install --no-deps --only-binary ":all:" --index-url https://test.pypi.org/simple/ "suitesparse-graphblas>=7.4.3" - elif [[ ${{ steps.sourcetype.outputs.selected }} == "source" ]]; then + elif [[ $SOURCETYPE == "source" ]]; then # Add --pre if installing a pre-release - python -m pip install --no-deps --no-binary suitesparse-graphblas suitesparse-graphblas${psgver} + python -m pip install --no-deps --no-binary suitesparse-graphblas "suitesparse-graphblas${PSGVER}" # Add the below line to the conda install command above if installing from test.pypi.org # ${{ steps.sourcetype.outputs.selected == 'source' && 'setuptools setuptools-git-versioning wheel cython' || '' }} \ # python -m pip install --no-deps --no-build-isolation --no-binary suitesparse-graphblas --index-url https://test.pypi.org/simple/ suitesparse-graphblas==7.4.3.3 - elif [[ ${{ steps.sourcetype.outputs.selected }} == "upstream" ]]; then + elif [[ $SOURCETYPE == "upstream" ]]; then python -m pip install --no-deps git+https://github.com/GraphBLAS/python-suitesparse-graphblas.git@main#egg=suitesparse-graphblas fi python -m pip install --no-deps -e . @@ -245,6 +317,9 @@ jobs: - name: JIT diagnostics run: python scripts/jit_diagnostics.py || true - name: Unit tests + # A hang (e.g. two libomp copies deadlocking in an OpenMP barrier on macOS) would + # otherwise hold the runner for GitHub's 6-hour limit. + timeout-minutes: 120 run: | A=${{ needs.rngs.outputs.mapnumpy == 'A' || '' }} ; B=${{ needs.rngs.outputs.mapnumpy == 'B' || '' }} C=${{ needs.rngs.outputs.mapnumpy == 'C' || '' }} ; D=${{ needs.rngs.outputs.mapnumpy == 'D' || '' }} @@ -277,6 +352,7 @@ jobs: coverage run -m pytest --color=yes --randomly -v ${args} \ ${{ matrix.slowtask == 'pytest_normal' && '--runslow' || '' }} - name: Unit tests (bizarro scalars) + timeout-minutes: 120 run: | # Run tests again with Scalars being C scalars by default find graphblas -type f -name "*.py" -print0 | xargs -0 sed -i -s \ @@ -317,6 +393,7 @@ jobs: git checkout . # Undo changes to scalar default - name: Miscellaneous tests if: matrix.slowtask == 'pytest_normal' + timeout-minutes: 30 run: | # Test (and cover) automatic initialization coverage run -a graphblas/tests/test_auto_init.py @@ -340,11 +417,24 @@ jobs: # coverage run -a -m pytest --color=yes -x --no-mapnumpy -k test_npmonoid graphblas/tests/test_numpyops.py --runslow - name: More tests for coverage if: matrix.slowtask == 'notebooks' && matrix.os == 'windows-latest' + timeout-minutes: 30 run: | # We use 'notebooks' slow task b/c it should have numba installed coverage run -a -m pytest --color=yes --runslow --no-mapnumpy -p no:randomly -v -k 'test_commutes or test_bool_doesnt_get_too_large or test_npbinary or test_npmonoid or test_npsemiring' coverage run -a -m pytest --color=yes --runslow --mapnumpy -p no:randomly -k 'test_bool_doesnt_get_too_large or test_npunary or test_binaryop_monoid_numpy' coverage run -a -m pytest --color=yes -x --no-mapnumpy --runslow -k test_binaryop_attributes_numpy graphblas/tests/test_op.py + - name: JIT tests for coverage + if: matrix.slowtask == 'notebooks' && matrix.os != 'windows-latest' + timeout-minutes: 30 + run: | + # Most of test_ssjit.py needs the suitesparse backend, --runslow, Numba and a + # GraphBLAS that can JIT-compile, all in one pytest run. In one run in four or five, + # no job draws all four, so its coverage came and went. 'notebooks' jobs install + # Numba unless the picker rules it out. Windows is left out: conda-forge's GraphBLAS + # there names the build machine's compiler, so it never compiles. The tests skip + # what a draw cannot do: the JIT tests on psg's wheels (built without the JIT) and + # on GraphBLAS 7, the UDT tests without Numba. + coverage run -a -m pytest --color=yes --runslow --backend=suitesparse -p no:randomly -v graphblas/tests/test_ssjit.py - name: Auto-generated code check if: matrix.slowtask == 'pytest_bizarro' run: | @@ -357,7 +447,7 @@ jobs: coverage xml coverage report --show-missing - name: codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: # Codecov takes tokenless uploads only from forks. A PR from a branch of this # repository, or a push to main, is refused with "Token required because @@ -365,6 +455,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Notebooks Execution check if: matrix.slowtask == 'notebooks' + timeout-minutes: 30 run: | # Run notebooks only if numba is installed if python -c 'import numba' 2> /dev/null ; then diff --git a/.github/zizmor.yml b/.github/zizmor.yml deleted file mode 100644 index 61f32c2e0..000000000 --- a/.github/zizmor.yml +++ /dev/null @@ -1,16 +0,0 @@ -rules: - use-trusted-publishing: - # TODO: we should update to use trusted publishing - ignore: - - publish_pypi.yml - excessive-permissions: - # It is probably good practice to use narrow permissions - ignore: - - debug.yml - - imports.yml - - publish_pypi.yml - - test_and_build.yml - template-injection: - # We use templates pretty heavily - ignore: - - test_and_build.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index def06f9d5..8515f7f41 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,9 @@ ci: autoupdate_commit_msg: "chore: update pre-commit hooks" autofix_commit_msg: "style: pre-commit fixes" # pysentry downloads advisory databases at run time, and pre-commit.ci - # runs hooks without network access, so it can only run locally. + # runs hooks without network access, so it runs locally and in security.yml. + # zizmor does run on pre-commit.ci, offline, which skips its online audits; + # security.yml runs those. skip: [pylint, no-commit-to-branch, pysentry] fail_fast: false default_language_version: @@ -41,7 +43,7 @@ repos: - id: name-tests-test args: ["--pytest-test-first"] - repo: https://github.com/abravalheri/validate-pyproject - rev: "0.26" + rev: v0.27 hooks: - id: validate-pyproject name: Validate pyproject.toml @@ -54,12 +56,12 @@ repos: # We can probably remove `isort` if we come to trust `ruff --fix`, # but we'll need to figure out the configuration to do this in `ruff` - repo: https://github.com/pycqa/isort - rev: 9.0.1 + rev: 9.0.2 hooks: - id: isort # Let's keep `pyupgrade` even though `ruff --fix` probably does most of it - repo: https://github.com/asottile/pyupgrade - rev: v3.21.2 + rev: v3.22.0 hooks: - id: pyupgrade args: [--py311-plus] @@ -69,27 +71,27 @@ repos: - id: auto-walrus args: [--line-length, "100"] - repo: https://github.com/psf/black-pre-commit-mirror - rev: 26.5.1 + rev: 26.10.1 hooks: - id: black - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.8 + rev: v0.17.3 hooks: - id: ruff-check args: [--fix-only, --show-fixes] # Let's keep `flake8` even though `ruff` does much of the same. # `flake8-bugbear` and `flake8-simplify` have caught things missed by `ruff`. - repo: https://github.com/PyCQA/flake8 - rev: 7.3.0 + rev: 7.5.0 hooks: - id: flake8 args: ["--config=.flake8"] additional_dependencies: &flake8_dependencies # These versions need updated manually - - flake8==7.3.0 - - flake8-bugbear==26.9.9 - - flake8-simplify==0.30.0 + - flake8==7.5.0 + - flake8-bugbear==26.10.27 + - flake8-simplify==0.31.1 - repo: https://github.com/codespell-project/codespell rev: v2.4.3 hooks: @@ -99,7 +101,7 @@ repos: - tomli; python_version<'3.11' files: ^(graphblas|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.8 + rev: v0.17.3 hooks: - id: ruff-check # - id: ruff-format # we prefer black; uncomment to compare with ruff's formatter @@ -122,7 +124,7 @@ repos: hooks: - id: shellcheck - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.7 + rev: v3.9.10 hooks: - id: prettier args: [--prose-wrap=preserve] @@ -146,8 +148,12 @@ repos: rev: v1.7.12 hooks: - id: actionlint + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.32.0 + hooks: + - id: zizmor - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.38.0 + rev: 0.39.0 hooks: - id: check-dependabot - id: check-github-workflows @@ -156,15 +162,22 @@ repos: rev: v1.38.0 hooks: - id: yamllint - # - repo: https://github.com/woodruffw/zizmor-pre-commit - # rev: v1.14.2 - # hooks: - # - id: zizmor - repo: https://github.com/pysentry/pysentry-pre-commit rev: v0.5.0 hooks: - id: pysentry - args: ["--format=human", "--fail-on=low", "--sources=pypa,osv,pypi"] + # Run when pyproject.toml or this file changes rather than at every commit, where + # it fails offline. security.yml also runs it weekly. + always_run: false + files: ^(pyproject\.toml|\.pre-commit-config\.yaml)$ + # Audit the floors in pyproject.toml. Without `--direct-only`, pysentry resolves the + # newest versions with uv (which the hook installs), and no longer sees a floor that + # allows a vulnerable release. + args: + - --format=human + - --fail-on=low + - --sources=pypa,osv,pypi + - --direct-only - repo: meta hooks: - id: check-hooks-apply diff --git a/graphblas/core/operator/udf.py b/graphblas/core/operator/udf.py index ab8144fc2..30b3e7ff4 100644 --- a/graphblas/core/operator/udf.py +++ b/graphblas/core/operator/udf.py @@ -39,7 +39,7 @@ def _bool_to_int8(dtype): numba/numba#5395. Routing BOOL through INT8 sidesteps that; GraphBLAS coerces back at the cfunc boundary. - MAINT 2026-05-24: still hits on Numba 0.65. Re-test periodically and + MAINT 2026-10-18: still hits on Numba 0.68. Re-test periodically and drop the INT8 routing when upstream is fixed. """ return INT8 if dtype == BOOL else dtype @@ -306,7 +306,7 @@ def _input_operand(dtype, var): if dtype == BOOL: # Numba can't compile bool ptrs (numba/numba#5395); expose them # as int8 and cast on deref. - # MAINT 2026-05-24: still hits on Numba 0.65; re-test periodically. + # MAINT 2026-10-18: still hits on Numba 0.68; re-test periodically. return "", f"bool({var}_ptr[0])", nt.CPointer(INT8.numba_type) return "", f"{var}_ptr[0]", nt.CPointer(dtype.numba_type) diff --git a/pyproject.toml b/pyproject.toml index 9c9701dde..cf0251968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: 3 :: Only", "Intended Audience :: Developers", "Intended Audience :: Other Audience", @@ -62,7 +63,7 @@ dependencies = [ # https://peps.python.org/pep-0771/ (still draft as of 2026-03) # Use e.g. "python-graphblas[suitesparse]" or "python-graphblas[default]" instead "suitesparse-graphblas >=7.4.0.0", - "numba >=0.57; python_version<'3.15'", # make optional where numba is not supported + "numba >=0.57; python_version<'3.16'", # make optional where numba is not supported ] [project.urls] @@ -83,11 +84,11 @@ suitesparse-udf = [ # udf requires numba repr = ["python-graphblas[pandas]"] io = [ "python-graphblas[networkx,scipy]", - "python-graphblas[numba]; python_version<'3.15'", # make optional where numba is not supported + "python-graphblas[numba]; python_version<'3.16'", # make optional where numba is not supported "awkward >=2.0", # sparse needs numba, so it stays optional and takes numba's upper guard from above: # it has to drop out wherever numba does. - "sparse >=0.14; python_version<'3.15'", + "sparse >=0.14; python_version<'3.16'", # fast-matrix-market is no longer maintained; last supported Python is 3.12 ] viz = ["python-graphblas[networkx,scipy]", "matplotlib >=3.6"] @@ -99,11 +100,11 @@ datashade = [ # datashade requires numba test = ["python-graphblas[suitesparse,pandas,scipy]", "packaging >=21", "pytest >=6.2", "tomli >=1"] default = [ "python-graphblas[suitesparse,pandas,scipy]", - "python-graphblas[numba]; python_version<'3.15'", # make optional where numba is not supported + "python-graphblas[numba]; python_version<'3.16'", # make optional where numba is not supported ] all = [ "python-graphblas[default,io,viz,test]", - "python-graphblas[datashade]; python_version<'3.15'", # make optional, b/c datashade needs numba + "python-graphblas[datashade]; python_version<'3.16'", # make optional, b/c datashade needs numba ] [tool.setuptools] @@ -138,6 +139,10 @@ dirty_template = "{tag}+{ccount}.g{sha}.dirty" [tool.black] line-length = 100 +# MAINT: add "py315" once a black that knows it (26.5 or later) installs on Python 3.15 +# from conda-forge (2026-10-19: its pytokens >=0.4 has no 3.15 build, so 3.15 gets black +# 26.1, which rejects "py315" and stops CI's auto-generated code check). Listing it also +# makes any older black refuse to run on this repository. target-version = ["py311", "py312", "py313", "py314"] [tool.isort] diff --git a/scripts/audit_conda_versions.py b/scripts/audit_conda_versions.py index 0d2fe4c16..5b837622d 100755 --- a/scripts/audit_conda_versions.py +++ b/scripts/audit_conda_versions.py @@ -55,6 +55,9 @@ # of these drifts, the matching rule in `apply_constraints` needs another look. # Each entry is (dep, version pin, field, expected constraint). DOCUMENTED_METADATA = [ + ("networkx", "3.7", "constrains", "numpy >=1.25"), + ("networkx", "3.7", "constrains", "scipy >=1.11.2"), + ("networkx", "3.7", "constrains", "pandas >=2.0"), ("networkx", "3.6", "constrains", "numpy >=1.25"), ("networkx", "3.6", "constrains", "scipy >=1.11.2"), ("networkx", "3.6", "constrains", "pandas >=2.0"), diff --git a/scripts/check_versions.sh b/scripts/check_versions.sh index f702f2f41..e99cb7ca6 100755 --- a/scripts/check_versions.sh +++ b/scripts/check_versions.sh @@ -10,20 +10,20 @@ # scripts/audit_pypi_versions.py and scripts/audit_conda_versions.py ask the same # questions automatically, and also check the pools and tables in ci_pick_versions.py # against what is published. The "Audit dependency versions" workflow runs both weekly. -conda search 'flake8-bugbear[channel=conda-forge]>=26.9.9' -conda search 'flake8-simplify[channel=conda-forge]>=0.30.0' +conda search 'flake8-bugbear[channel=conda-forge]>=26.10.27' +conda search 'flake8-simplify[channel=conda-forge]>=0.31.1' conda search 'numpy[channel=conda-forge]>=2.5' # ci_pick_versions.py caps an unpinned pandas below 3.1 for now (see its MAINT note). conda search 'pandas[channel=conda-forge]>=3.0' conda search 'scipy[channel=conda-forge]>=1.18' -conda search 'networkx[channel=conda-forge]>=3.6' -conda search 'awkward[channel=conda-forge]>=2.13' +conda search 'networkx[channel=conda-forge]>=3.7' +conda search 'awkward[channel=conda-forge]>=2.14' conda search 'sparse[channel=conda-forge]>=0.19' # fast_matrix_market is deprecated (no longer maintained; last supported Python is 3.12) -conda search 'numba[channel=conda-forge]>=0.67' +conda search 'numba[channel=conda-forge]>=0.68' conda search 'pyyaml[channel=conda-forge]>=6.0' # matplotlib is installed unpinned for the notebooks job; ci_pick_versions.py caps it # below 3.11 when the numpy pick predates matplotlib's numpy >=1.25 runtime floor. conda search 'matplotlib-base[channel=conda-forge]>=3.11' -conda search 'python-suitesparse-graphblas[channel=conda-forge]>=10.5.0' +conda search 'python-suitesparse-graphblas[channel=conda-forge]>=10.5.2' # conda search 'python[channel=conda-forge]>=3.11 *pypy*' diff --git a/scripts/ci_pick_versions.py b/scripts/ci_pick_versions.py index 8c7d4786c..34f76e9ee 100755 --- a/scripts/ci_pick_versions.py +++ b/scripts/ci_pick_versions.py @@ -24,7 +24,7 @@ # # When updating versions here, also update scripts/check_versions.sh # -# To add a Python version (3.15, say): add it to `choices` in the "RNG for Python +# To add a Python version (3.16, say): add it to `choices` in the "RNG for Python # version" step of test_and_build.yml, give it a row in every table keyed by Python # below (`check_python_pools` fails the run if you miss one), and revisit the floors # in `apply_constraints`. Those floors are written as `_ver(pyver) >= (3, N)`, so a @@ -32,12 +32,14 @@ # enough to keep CI honest, but usually still too loose to install, so check each dep. # --------------------------------------------------------------------------- -# numpy 2.5 requires Python >=3.12, so py3.11 stops at 2.4. +# numpy 2.5 requires Python >=3.12, so py3.11 stops at 2.4. conda-forge's first py3.15 +# builds are numpy 2.5.2, which the "2.5" pin resolves to. NUMPY_VERSIONS = { "3.11": ["1.24", "1.25", "1.26", "2.0", "2.1", "2.2", "2.3", "2.4", ""], "3.12": ["1.26", "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", ""], "3.13": ["2.1", "2.2", "2.3", "2.4", "2.5", ""], "3.14": ["2.3", "2.4", "2.5", ""], + "3.15": ["2.5", ""], } # Deps that depend on numpy version (1.x vs 2.x path). @@ -99,16 +101,22 @@ "3.11": ["2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9", ""], "3.12": ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9", ""], }, - "2.x": ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12", "2.13", ""], + "2.x": ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12", "2.13", "2.14", ""], } NUMBA_VERSIONS = { "1.x": ["0.57", "0.58", "0.59", "0.60", "0.61", ""], - "2.x": ["0.62", "0.63", "0.64", "0.65", "0.66", "0.67", ""], + "2.x": ["0.62", "0.63", "0.64", "0.65", "0.66", "0.67", "0.68", ""], } # Oldest numba that supports each Python. -NUMBA_MIN_PYTHON = {"3.11": (0, 57), "3.12": (0, 59), "3.13": (0, 61), "3.14": (0, 63)} +NUMBA_MIN_PYTHON = { + "3.11": (0, 57), + "3.12": (0, 59), + "3.13": (0, 61), + "3.14": (0, 63), + "3.15": (0, 68), +} # The numpy ceiling each numba pin declares. numba caps `numpy =3.12, so py3.11 stops at 3.6. NETWORKX_VERSIONS = { "3.11": ["2.8", "3.0", "3.1", "3.2", "3.3", "3.4", "3.5", "3.6", ""], - "3.12": ["3.2", "3.3", "3.4", "3.5", "3.6", ""], - "3.13": ["3.4", "3.5", "3.6", ""], - "3.14": ["3.6", ""], + "3.12": ["3.2", "3.3", "3.4", "3.5", "3.6", "3.7", ""], + "3.13": ["3.4", "3.5", "3.6", "3.7", ""], + "3.14": ["3.6", "3.7", ""], + "3.15": ["3.6", "3.7", ""], } PYYAML_VERSIONS = { @@ -145,6 +156,7 @@ "3.12": ["6.0", ""], "3.13": ["6.0", ""], "3.14": ["6.0", ""], + "3.15": ["6.0", ""], } # sparse is noarch and caps neither Python nor numpy: every release here declares numba @@ -161,6 +173,7 @@ "3.12": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], "3.13": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], "3.14": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], + "3.15": ["0.14", "0.15", "0.16", "0.17", "0.18", "0.19", ""], } # PSG versions to pair with numpy 1.x (only reachable on py3.11/py3.12, since @@ -182,8 +195,10 @@ ], "3.12": ["8.2.0.1", "8.2.1.0"], }, + # The picker does not know the OS, and 7.4.3.2's wheels are manylinux x86_64 only, + # so a "wheel" draw of it would fail to install on every other runner. "wheel": { - "3.11": ["7.4.3.2", "8.0.2.1", "8.2.0.1", "8.2.1.0"], + "3.11": ["8.0.2.1", "8.2.0.1", "8.2.1.0"], "3.12": ["8.2.0.1", "8.2.1.0"], }, } @@ -200,6 +215,8 @@ "10.4.0.0", "10.4.1.0", "10.5.0.0", + "10.5.1.1", + "10.5.2.0", ] # Oldest psg release that installs on a given Python, by where psg comes from. @@ -211,9 +228,14 @@ # can't reach back as far. "source" builds the sdist, and the sdists before 10.3.1.0 # predate psg's py3.14 support (they still declare requires-python >=3.9), so hold # them to the wheel floor rather than test a build psg never claimed works. -# MAINT: 2026-09-11 checked conda-forge build strings and PyPI wheel tags for psg 9-10 +# +# py3.15 starts at 10.5.1.1 everywhere: it is psg's first release with py315 builds on +# conda-forge and cp315 wheels on PyPI, and its sdist is the first from after psg added +# Python 3.15 support. +# MAINT: 2026-10-28 checked conda-forge build strings and PyPI wheel tags for psg 9-10 PSG_MIN_PYTHON = { "3.14": {"conda-forge": "10.0.1.1", "wheel": "10.3.1.0", "source": "10.3.1.0"}, + "3.15": {"conda-forge": "10.5.1.1", "wheel": "10.5.1.1", "source": "10.5.1.1"}, } @@ -320,8 +342,10 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- scipy / Python version availability --- - # scipy <1.14 has no py3.13 builds; scipy <1.16 has no py3.14 builds - if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): + # scipy <1.14 has no py3.13 builds, <1.16 none for py3.14, and <1.18 none for py3.15 + if _ver(pyver) >= (3, 15) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 18): + v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 18)) + elif _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 16)) elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": v["scipy"] = _pick_scipy(pyver, npver, scipy_pool, floor=(1, 14)) @@ -332,8 +356,10 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- pandas constraints --- - # pandas <2.3 has no py3.14 builds - if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": + # pandas <2.3 has no py3.14 builds, and <3.0 none for py3.15 + if _ver(pyver) >= (3, 15) and v["pandas"] not in ("", "NA") and _ver(v["pandas"]) < (3, 0): + v["pandas"] = random.choice(["3.0", ""]) + elif _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": v["pandas"] = random.choice(["2.3", "3.0", ""]) # pandas 3.0 requires scipy >=1.14.1. It also requires numba >=0.60, which needs no @@ -367,8 +393,12 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- awkward / Python version availability --- - # awkward <2.7 has no py3.13 builds; awkward <2.8 has no py3.14 builds - if _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): + # awkward <2.7 has no py3.13 builds, <2.8 none for py3.14, and <2.13 none for py3.15 + # (awkward is noarch, but each release pins one awkward-cpp, and awkward-cpp 56 is the + # first with py315 builds) + if _ver(pyver) >= (3, 15) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 13): + v["awkward"] = random.choice([a for a in AWKWARD_VERSIONS["2.x"] if _ver(a) >= (2, 13)]) + elif _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): v["awkward"] = random.choice(["2.8", "2.9", ""]) elif _ver(pyver) >= (3, 13) and v["awkward"] == "2.6": v["awkward"] = random.choice(["2.7", "2.8", "2.9", ""]) @@ -388,14 +418,14 @@ def apply_constraints(v, pyver, scipy_pool, numba_pool): # --- networkx / numpy, scipy, pandas floors --- - # conda-forge networkx 3.5 and 3.6 declare `constrains: numpy >=1.25, scipy >=1.11.2, + # conda-forge networkx 3.5 through 3.7 declare `constrains: numpy >=1.25, scipy >=1.11.2, # pandas >=2.0`. conda enforces those against our pins, and since networkx is pinned too # the solve fails outright instead of backing off. Older pins are safe because a pin like # "3.4" can resolve down to 3.4.0, whose floors (numpy >=1.22, scipy >=1.9, pandas >=1.4) # sit at or below everything in our pools. Unpinned networkx is latest, so it gets the # 3.5+ floors. Compare scipy at the minor level: the "1.11" pin resolves to 1.11.4, which # clears the 1.11.2 floor. - # MAINT: 2026-08-04 conda-forge networkx 3.6.1 constrains numpy >=1.25, scipy >=1.11.2 + # MAINT: 2026-10-18 conda-forge networkx 3.6.1 and 3.7 constrain numpy >=1.25, scipy >=1.11.2 if _ver(v["networkx"]) >= (3, 5) and ( _ver(v["numpy"]) < (1, 25) or _ver(v["scipy"]) < (1, 11) or _ver(v["pandas"]) < (2, 0) ): @@ -685,8 +715,10 @@ def validate(v, pyver, source_type=None): errors.append(f"scipy {v['scipy']} requires numpy <{cap}, got {v['numpy'] or 'latest'}") # scipy Python availability - if _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): - errors.append(f"scipy {v['scipy']} has no py{pyver} build") + if _ver(pyver) >= (3, 15) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 18): + errors.append(f"scipy {v['scipy']} has no py{pyver} build (needs >=1.18)") + elif _ver(pyver) >= (3, 14) and v["scipy"] not in ("", "NA") and _ver(v["scipy"]) < (1, 16): + errors.append(f"scipy {v['scipy']} has no py{pyver} build (needs >=1.16)") elif _ver(pyver) >= (3, 13) and v["scipy"] == "1.13": errors.append(f"scipy 1.13 has no py{pyver} build") if v["scipy"] and _ver(pyver) < (floor := SCIPY_MIN_PYTHON.get(v["scipy"], (0,))): @@ -694,7 +726,9 @@ def validate(v, pyver, source_type=None): errors.append(f"scipy {v['scipy']} requires Python >={need}, has no py{pyver} build") # pandas Python availability - if _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": + if _ver(pyver) >= (3, 15) and v["pandas"] not in ("", "NA") and _ver(v["pandas"]) < (3, 0): + errors.append(f"pandas {v['pandas']} has no py{pyver} build") + elif _ver(pyver) >= (3, 14) and v["pandas"] == "2.2": errors.append(f"pandas 2.2 has no py{pyver} build") # pandas 3.0 requirements (numba >=0.60 is unreachable; see `apply_constraints`) @@ -711,8 +745,10 @@ def validate(v, pyver, source_type=None): errors.append(f"pandas {v['pandas']} constrains scipy >=1.10.0, got {v['scipy']}") # awkward Python availability - if _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): - errors.append(f"awkward {v['awkward']} has no py{pyver} build") + if _ver(pyver) >= (3, 15) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 13): + errors.append(f"awkward {v['awkward']} has no py{pyver} build (needs >=2.13)") + elif _ver(pyver) >= (3, 14) and v["awkward"] not in ("", "NA") and _ver(v["awkward"]) < (2, 8): + errors.append(f"awkward {v['awkward']} has no py{pyver} build (needs >=2.8)") elif _ver(pyver) >= (3, 13) and v["awkward"] == "2.6": errors.append(f"awkward 2.6 has no py{pyver} build")