diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cf876f7..30cde65 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -31,3 +31,9 @@ updates: interval: monthly cooldown: default-days: 7 + - package-ecosystem: "github-actions" + directory: "/actions/python-lambda-zip" + schedule: + interval: monthly + cooldown: + default-days: 7 diff --git a/README.md b/README.md index 8e8141f..21d31a4 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,21 @@ avoid breaking things. If you feel lucky you might use `v0` branch. # actions Number of helpful actions for GitHub Actions -* [build-push-ecr](./actions/build-push-ecr/README.md) +* [build-push-ecr](./actions/build-push-ecr/README.md): + builds and pushes an image to AWS ECR using buildx -* [download-private-asset](./actions/download-private-asset/README.md) +* [download-private-asset](./actions/download-private-asset/README.md): + installs a release asset from a private GitHub repository + +* [python-lambda-zip](./actions/python-lambda-zip/README.md): + builds a zip deployment package for a Python AWS Lambda using runtime-compatible + wheels, and checks that it loads in the AWS Lambda base image + +* [terraform](./actions/terraform/README.md): + Terraform format linting, required-tag checks and tfsec security scanning + +* [upgrade-erlang-elixir](./actions/upgrade-erlang-elixir/README.md): + resolves target Erlang/Elixir versions, updates them in CI files and opens a pull request # reusable workflows Workflows that a caller repo runs with `uses` at job level. Intentionally sit at diff --git a/actions/python-lambda-zip/README.md b/actions/python-lambda-zip/README.md new file mode 100644 index 0000000..7ee2e68 --- /dev/null +++ b/actions/python-lambda-zip/README.md @@ -0,0 +1,67 @@ +# Python Lambda zip + +Builds a zip-file deployment package for a Python AWS Lambda and checks that it actually loads in +the AWS Lambda runtime. + +Dependencies with compiled extensions (`cryptography`, `numpy`, ...) ship platform-specific wheels. +A plain `pip install` on a GitHub runner picks wheels built for the runner's (new) glibc, which the +Lambda runtime may be too old to load, giving errors like +``version `GLIBC_2.28' not found`` only at import time in AWS. This action + +1. installs the dependencies with `pip --platform ... --only-binary=:all:`, so that only wheels + compatible with the Lambda runtime are used, and +2. imports your modules inside the matching `public.ecr.aws/lambda/python` image, so that any + remaining problem fails the build instead of the deployment. + +The action does not upload the zip anywhere; use the `zip-path` output (or `output-dir`) for that. + +## Usage + +```yaml +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - run: pip install pipenv + - uses: gameanalytics/github/actions/python-lambda-zip@v0 + with: + python-version: '3.11' + source-dir: app + requirements-command: pipenv requirements + smoke-test-glob: 'lambda_*.py' +``` + +Run it in your pull request workflow too, not only on releases, to find problems before release. +The runner needs `pip`, `zip`, `unzip` and Docker, which `ubuntu-latest` has. + +## Inputs + +| Input | Default | Description | +|---|---|---| +| `python-version` | (required) | Python version of the Lambda runtime, e.g. `3.11` | +| `architecture` | `x86_64` | `x86_64` or `arm64` (arm64 is smoke tested using QEMU emulation) | +| `source-dir` | `src` | Directory with the Lambda code; its contents end up at the root of the zip | +| `requirements-file` | `requirements.txt` | Fully resolved requirements file | +| `requirements-command` | | Command printing the requirements, e.g. `pipenv requirements`; used instead of the file when set | +| `smoke-test-glob` | `*.py` | Modules (relative to the zip root) to import; build fails if nothing matches | +| `output-dir` | `build` | Where to put the zip | +| `zip-name` | `package.zip` | File name of the zip | + +Output: `zip-path`, the absolute path of the zip. + +## How `python-version` maps to wheels and image + +| Python version | Lambda OS | glibc | Wheels installed | Smoke test image | +|---|---|---|---|---| +| 3.9 - 3.11 | Amazon Linux 2 | 2.26 | `manylinux2014` | `public.ecr.aws/lambda/python:` | +| 3.12 and later | Amazon Linux 2023 | 2.34 | `manylinux_2_28` | `public.ecr.aws/lambda/python:` | + +The dependencies are installed with `--no-deps`, so the requirements must already be fully +resolved, as with a lock file. If a dependency has no compatible wheel (only an sdist, or only +newer manylinux wheels), the build fails instead of building something that breaks at runtime. + +When a new Lambda runtime moves to a newer OS, update the mapping in `build.sh`. diff --git a/actions/python-lambda-zip/action.yml b/actions/python-lambda-zip/action.yml new file mode 100644 index 0000000..35b1f9e --- /dev/null +++ b/actions/python-lambda-zip/action.yml @@ -0,0 +1,67 @@ +name: Python Lambda zip +description: > + Builds a zip-file deployment package for a Python AWS Lambda, using wheels compatible with the + Lambda runtime, and checks that it can be imported inside the AWS Lambda base image. + +inputs: + python-version: + description: > + Python version of the Lambda runtime, e.g. '3.11'. Determines the Lambda base image used for + the smoke test and which manylinux wheels are installed. + required: true + architecture: + description: "Lambda architecture: 'x86_64' or 'arm64'" + required: false + default: 'x86_64' + source-dir: + description: 'Directory with the Lambda code. Its contents end up at the root of the zip.' + required: false + default: 'src' + requirements-file: + description: 'Fully resolved requirements file, e.g. generated from a lock file.' + required: false + default: 'requirements.txt' + requirements-command: + description: > + Shell command that prints the requirements to stdout, e.g. 'pipenv requirements'. Used + instead of requirements-file when set. The tool it needs must already be installed. + required: false + smoke-test-glob: + description: > + Glob, relative to the root of the zip, of the modules to import in the smoke test. + The build fails if nothing matches. + required: false + default: '*.py' + output-dir: + description: 'Directory to put the zip in' + required: false + default: 'build' + zip-name: + description: 'File name of the zip' + required: false + default: 'package.zip' + +outputs: + zip-path: + description: 'Absolute path of the built zip' + value: ${{ steps.build.outputs.zip-path }} + +runs: + using: composite + steps: + - name: Set up QEMU + if: inputs.architecture == 'arm64' + uses: docker/setup-qemu-action@v4.4.0 + - name: Build zip and smoke test it + id: build + shell: bash + env: + PYTHON_VERSION: ${{ inputs.python-version }} + ARCHITECTURE: ${{ inputs.architecture }} + SOURCE_DIR: ${{ inputs.source-dir }} + REQUIREMENTS_FILE: ${{ inputs.requirements-file }} + REQUIREMENTS_COMMAND: ${{ inputs.requirements-command }} + SMOKE_TEST_GLOB: ${{ inputs.smoke-test-glob }} + OUTPUT_DIR: ${{ inputs.output-dir }} + ZIP_NAME: ${{ inputs.zip-name }} + run: ${{ github.action_path }}/build.sh diff --git a/actions/python-lambda-zip/build.sh b/actions/python-lambda-zip/build.sh new file mode 100755 index 0000000..8ab1925 --- /dev/null +++ b/actions/python-lambda-zip/build.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Builds a Lambda deployment zip for the given Python version and architecture, +# then imports it inside the matching AWS Lambda base image as a smoke test. +# Configured through environment variables, see action.yml. +set -euo pipefail + +: "${PYTHON_VERSION:?}" "${ARCHITECTURE:?}" "${SOURCE_DIR:?}" "${OUTPUT_DIR:?}" "${ZIP_NAME:?}" +: "${SMOKE_TEST_GLOB:?}" + +if [[ ! "$PYTHON_VERSION" =~ ^3\.[0-9]+$ ]]; then + echo "python-version must look like '3.11', got '$PYTHON_VERSION'" >&2 + exit 1 +fi +minor="${PYTHON_VERSION#3.}" + +# Lambda runtimes up to python3.11 run on Amazon Linux 2 (glibc 2.26), so only +# manylinux2014 (glibc 2.17) wheels can be loaded. python3.12 and later run on +# Amazon Linux 2023 (glibc 2.34), which can load manylinux_2_28 wheels. +if (( minor <= 11 )); then + manylinux=manylinux2014 +else + manylinux=manylinux_2_28 +fi + +case "$ARCHITECTURE" in + x86_64) docker_platform=linux/amd64 ;; + arm64) docker_platform=linux/arm64 ;; + *) echo "architecture must be x86_64 or arm64, got '$ARCHITECTURE'" >&2; exit 1 ;; +esac +arch=$([[ $ARCHITECTURE == arm64 ]] && echo aarch64 || echo x86_64) +# Also accept manylinux2014 wheels, which pip does not do implicitly for newer tags +wheel_platforms=(--platform "manylinux2014_${arch}") +[[ $manylinux == manylinux2014 ]] || wheel_platforms+=(--platform "${manylinux}_${arch}") +image="public.ecr.aws/lambda/python:${PYTHON_VERSION}" + +stage="$(mktemp -d)" +trap 'rm -rf "$stage"' EXIT +mkdir -p "$stage/package" "$OUTPUT_DIR" +zip_path="$(cd "$OUTPUT_DIR" && pwd)/$ZIP_NAME" +rm -f "$zip_path" + +if [[ -n "${REQUIREMENTS_COMMAND:-}" ]]; then + bash -c "$REQUIREMENTS_COMMAND" > "$stage/requirements.txt" +else + cp "$REQUIREMENTS_FILE" "$stage/requirements.txt" +fi + +# --no-deps because the requirements are expected to be fully resolved (a lock file) +echo "::group::Install dependencies for $manylinux $arch, Python $PYTHON_VERSION" +python3 -m pip install -r "$stage/requirements.txt" --no-deps -t "$stage/package" \ + "${wheel_platforms[@]}" --python-version "$PYTHON_VERSION" \ + --implementation cp --only-binary=:all: +echo "::endgroup::" + +cp -r "$SOURCE_DIR"/. "$stage/package/" +(cd "$stage/package" && zip -qr "$zip_path" .) +echo "Built $zip_path" + +mkdir -p "$stage/smoke" +unzip -q "$zip_path" -d "$stage/smoke" +echo "::group::Smoke test in $image ($docker_platform)" +docker run --rm --platform "$docker_platform" --entrypoint python \ + -v "$stage/smoke:/var/task:ro" -w /var/task -e GLOB="$SMOKE_TEST_GLOB" "$image" \ + -c "import glob, importlib, os +mods = sorted(f[:-3] for f in glob.glob(os.environ['GLOB'])) +assert mods, 'no modules matched ' + os.environ['GLOB'] +for m in mods: + importlib.import_module(m) + print('imported', m)" +echo "::endgroup::" + +echo "zip-path=$zip_path" >> "${GITHUB_OUTPUT:-/dev/null}"