Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions .github/workflows/audit_versions.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Checks the CI dependency pools in scripts/ci_pick_versions.py against the world.
#
# `ci_pick_versions.py --validate` runs in pre-commit and proves the picker agrees with
# the constraints it documents. It cannot notice when a constraint has drifted from
# reality, because that happens when a package releases, not when we commit. So this
# workflow runs weekly, and on changes to the picker or the audits themselves.
#
# A failure here does not mean CI is broken right now. It means a pool or a table no
# longer matches what PyPI or conda-forge publishes, and the next random pick that
# lands on the stale combination will fail for reasons that look unrelated.

name: Audit dependency versions

on:
schedule:
# Mondays at 06:00 UTC, before most of the week's PRs
- cron: "0 6 * * 1"
workflow_dispatch:
pull_request:
paths:
- scripts/ci_pick_versions.py
- scripts/audit_pypi_versions.py
- scripts/audit_conda_versions.py
- .github/workflows/audit_versions.yml
push:
branches:
- main
paths:
- scripts/ci_pick_versions.py
- scripts/audit_pypi_versions.py
- scripts/audit_conda_versions.py
- .github/workflows/audit_versions.yml

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
pypi:
name: Audit against PyPI
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- run: python -m pip install --upgrade pip packaging
- name: Stress-test the picker
run: python scripts/ci_pick_versions.py --validate
- name: Audit pools against PyPI
run: python scripts/audit_pypi_versions.py --samples 200

conda:
name: Audit against conda-forge
runs-on: ubuntu-latest
# conda-forge is what CI installs from, so this is the one that binds. It needs a
# conda to search with, which is why it is a separate, slower job.
defaults:
run:
shell: bash -el {0}
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: conda-incubator/setup-miniconda@v3
with:
auto-update-conda: true
python-version: "3.12"
channels: conda-forge
conda-remove-defaults: true
channel-priority: strict
activate-environment: audit
- run: python -m pip install --upgrade pip packaging
- name: Audit pools against conda-forge
run: python scripts/audit_conda_versions.py --samples 100
10 changes: 9 additions & 1 deletion .github/workflows/test_and_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,15 @@ jobs:
# Pick random compatible dependency versions using the version picker script.
# See scripts/ci_pick_versions.py for version pools and constraint logic.
pyver=$(python -c "v='${{ steps.pyver.outputs.selected }}'.split('.');print(f'{v[0]}.{v[1]}')")
eval "$(python scripts/ci_pick_versions.py --python "$pyver" --source ${{ steps.sourcetype.outputs.selected }})"
# Assign first, then eval. `eval "$(cmd)"` reports eval's status, not cmd's, and
# this step's shell has no `-e`, so a picker that exits non-zero (a pool table it
# rejects, say) would leave every *ver variable unset and quietly install the
# latest of everything: a green run that tested nothing it claims to have tested.
if ! picks=$(python scripts/ci_pick_versions.py --python "$pyver" --source ${{ steps.sourcetype.outputs.selected }}) ; then
echo "::error::ci_pick_versions.py failed; refusing to install unpinned dependencies"
exit 1
fi
eval "$picks"

# Set up psg conda package variable (conda-forge installs via conda, others via pip).
# For conda-forge, psg must always be installed; an empty psgver just means "latest".
Expand Down
38 changes: 23 additions & 15 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,23 +42,20 @@ repos:
- id: name-tests-test
args: ["--pytest-test-first"]
- repo: https://github.com/abravalheri/validate-pyproject
rev: v0.25
rev: "0.26"
hooks:
- id: validate-pyproject
name: Validate pyproject.toml
# I don't yet trust ruff to do what autoflake does
- repo: https://github.com/PyCQA/autoflake
rev: v2.3.3
rev: v2.4.0
hooks:
- id: autoflake
args: [--in-place]
# We can probably remove `isort` if we come to trust `ruff --fix`,
# but we'll need to figure out the configuration to do this in `ruff`
- repo: https://github.com/pycqa/isort
# 9.0 is still in alpha (9.0.0a3) and a pre-release pin defeats
# pre-commit.ci's autoupdate "avoid prereleases" logic; stay on the last
# stable 8.x until 9.0 GAs.
rev: 8.0.1
rev: 9.0.1
hooks:
- id: isort
# Let's keep `pyupgrade` even though `ruff --fix` probably does most of it
Expand All @@ -73,12 +70,12 @@ repos:
- id: auto-walrus
args: [--line-length, "100"]
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 26.3.1
rev: 26.5.1
hooks:
- id: black
- id: black-jupyter
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.13
rev: v0.16.8
hooks:
- id: ruff-check
args: [--fix-only, --show-fixes]
Expand All @@ -92,18 +89,18 @@ repos:
additional_dependencies:
&flake8_dependencies # These versions need updated manually
- flake8==7.3.0
- flake8-bugbear==25.11.29
- flake8-bugbear==26.9.9
- flake8-simplify==0.30.0
- repo: https://github.com/codespell-project/codespell
rev: v2.4.2
rev: v2.4.3
hooks:
- id: codespell
types_or: [python, markdown, rst, toml, yaml]
additional_dependencies:
- tomli; python_version<'3.11'
files: ^(graphblas|docs)/
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.13
rev: v0.16.8
hooks:
- id: ruff-check
# - id: ruff-format # we prefer black; uncomment to compare with ruff's formatter
Expand All @@ -115,16 +112,18 @@ repos:
# `pyroma` may help keep our package standards up to date if best practices change.
# This is probably a "low value" check though and safe to remove if we want faster pre-commit.
- repo: https://github.com/regebro/pyroma
# 5.1 is still in beta (5.1b2) and a pre-release pin defeats pre-commit.ci's
# autoupdate "avoid prereleases" logic; stay on 5.0.1 until 5.1 GAs.
rev: "5.0.1"
hooks:
- id: pyroma
args: [-n, "10", .]
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: "v0.11.0.1"
rev: "v0.11.0.1-1"
hooks:
- id: shellcheck
- repo: https://github.com/rbubley/mirrors-prettier
rev: v3.8.3
rev: v3.9.7
hooks:
- id: prettier
args: [--prose-wrap=preserve]
Expand All @@ -149,7 +148,7 @@ repos:
hooks:
- id: actionlint
- repo: https://github.com/python-jsonschema/check-jsonschema
rev: 0.37.2
rev: 0.38.0
hooks:
- id: check-dependabot
- id: check-github-workflows
Expand All @@ -163,7 +162,7 @@ repos:
# hooks:
# - id: zizmor
- repo: https://github.com/pysentry/pysentry-pre-commit
rev: v0.4.6
rev: v0.5.0
hooks:
- id: pysentry
args: ["--format=human", "--fail-on=low", "--sources=pypa,osv,pypi"]
Expand Down Expand Up @@ -206,6 +205,15 @@ repos:
| scripts/find_aiisms\.sh
| \.pre-commit-config\.yaml
)$
# Stress-tests every combination the CI dependency picker can produce against
# the constraints it documents. ~3s for 160k combinations, and it only runs
# when the picker itself changes.
- id: ci-pick-versions
name: Validate CI dependency version picker
entry: python scripts/ci_pick_versions.py --validate
language: system
files: ^scripts/ci_pick_versions\.py$
pass_filenames: false
# Add `--hook-stage manual` to pre-commit command to run (very slow)
# It's probably better (and faster!) to simply run `pylint graphblas/some/file.py`
- id: pylint
Expand Down
12 changes: 11 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,14 @@ io = [
"python-graphblas[networkx,scipy]",
"python-graphblas[numba]; python_version<'3.15'", # make optional where numba is not supported
"awkward >=2.0",
"sparse >=0.14; python_version<'3.13'", # make optional, b/c sparse needs numba
# sparse needs numba, so it stays optional. numba gained py3.13 support in 0.61 and
# py3.14 in 0.63; sparse 0.16 is the first release that postdates py3.13, and 0.18 the
# first that postdates py3.14.
"sparse >=0.14; python_version<'3.13'",
"sparse >=0.16; python_version=='3.13'",
# Upper guard for the same reason as numba above: sparse depends on numba, so it has
# to drop out wherever numba does, not just trail it.
"sparse >=0.18; python_version>='3.14' and python_version<'3.15'",
# fast-matrix-market is no longer maintained; last supported Python is 3.12
]
viz = ["python-graphblas[networkx,scipy]", "matplotlib >=3.6"]
Expand Down Expand Up @@ -342,6 +349,7 @@ ignore = [
"PLR0911", # Too many return statements
"PLR0912", # Too many branches
"PLR0913", # Too many arguments to function call
"PLR0917", # Too many positional arguments (Stable since ruff 0.16; same call for PLR0913)
"PLR0915", # Too many statements
"PLR2004", # Magic number used in comparison, consider replacing magic with a constant variable
"PLW0603", # Using the global statement to update ... is discouraged (Note: yeah, discouraged, but too strict)
Expand All @@ -362,6 +370,7 @@ ignore = [

# Ignored categories
"C90", # mccabe (Too strict, but maybe we should make things less complex)
"CPY", # flake8-copyright (Stable since ruff 0.16; we don't put a header in every file)
"I", # isort (Should we replace `isort` with this?)
"ANN", # flake8-annotations (We don't use annotations yet)
"BLE", # flake8-blind-except (Maybe consider)
Expand Down Expand Up @@ -394,6 +403,7 @@ ignore = [
"NPY002", # numba doesn't support rng generator yet
]
"graphblas/core/utils.py" = ["PLE0302"] # `__set__` is used as a property
"graphblas/core/formatting.py" = ["ISC004"] # Adjacent literals form one HTML fragment
"scripts/ci_pick_versions.py" = ["S311"] # random.choice is intentional for CI version selection
"graphblas/ss/_core.py" = ["N999"] # We want _core.py to be underscopre
# Allow useless expressions, assert, pickle, RNG, print, no docstring, and yoda in tests
Expand Down
Loading
Loading